# https://quantpedia.com/strategies/rd-expenditures-and-stock-returns/
#
# The investment universe consists of stocks that are listed on NYSE NASDAQ or AMEX. At the end of April, for each stock in the universe, calculate a measure of total R&D expenditures in the past 5 years scaled by the firm’s Market cap (defined on page 7, eq. 1).
# Go long (short) on the quintile of firms with the highest (lowest) R&D expenditures relative to their Market Cap. Weight the portfolio equally and rebalance next year. The backtested performance of the paper is substituted by our more recent backtest in Quantconnect.

#region imports
from AlgorithmImports import *
from numpy import log, average
from scipy import stats
import numpy as np
#endregion

class RDExpendituresandStockReturns(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(1998, 1, 1)
        self.SetCash(100000)

        self.weight = {}
        self.coarse_count = 3000
        
        # R&D history.
        self.RD = {}
        self.rd_period = 5
        self.quantile = 5
        
        self.long = []
        self.short = []
        
        data = self.AddEquity('XLK', Resolution.Daily)
        data.SetLeverage(10)
        self.technology_sector = data.Symbol
      
        self.symbol = self.AddEquity('SPY', Resolution.Daily).Symbol
        
        self.selection_flag = True
        self.UniverseSettings.Resolution = Resolution.Daily
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)
        self.Schedule.On(self.DateRules.MonthEnd(self.symbol), self.TimeRules.AfterMarketOpen(self.symbol), self.Selection)
   
    def OnSecuritiesChanged(self, changes):
        for security in changes.AddedSecurities:
            security.SetLeverage(10)
            security.SetFeeModel(CustomFeeModel())
        
    def CoarseSelectionFunction(self, coarse):
        if not self.selection_flag:
            return Universe.Unchanged
        
        selected = [x.Symbol for x in coarse if x.HasFundamentalData and x.Price > 5]
        
        return selected

    def FineSelectionFunction(self, fine):
        fine = [x for x in fine if (x.FinancialStatements.IncomeStatement.ResearchAndDevelopment.TwelveMonths) and \
                                    (x.MarketCap != 0) and \
                                    ((x.SecurityReference.ExchangeId == "NYS") or (x.SecurityReference.ExchangeId == "NAS") or (x.SecurityReference.ExchangeId == "ASE"))]
                                    #and x.AssetClassification.MorningstarSectorCode == MorningstarSectorCode.Technology]
        
        top_by_market_cap = None
        if len(fine) > self.coarse_count:
            sorted_by_market_cap = sorted(fine, key = lambda x:x.MarketCap, reverse=True)
            top_by_market_cap = sorted_by_market_cap[:self.coarse_count]
        else:
            top_by_market_cap = fine
        
        fine_symbols = [x.Symbol for x in top_by_market_cap]
        ability = {}
        
        updated_flag = []  # updated this year already
        
        for stock in top_by_market_cap:
            symbol = stock.Symbol
            
            # prevent storing duplicated value for the same stock in one year
            if symbol not in updated_flag:
                
                # Update RD.
                if symbol not in self.RD:
                    self.RD[symbol] = RollingWindow[float](self.rd_period)
                #rd = stock.FinancialStatements.IncomeStatement.ResearchAndDevelopment.TwelveMonths
                #self.RD[symbol].Add(rd)
                
                if self.RD[symbol].IsReady:
                    coefs = np.array([1, 0.8, 0.6, 0.4, 0.2])
                    rds = np.array([x for x in self.RD[symbol]])
                    
                    rdc = sum(coefs * rds)
                    ability[stock] = rdc/stock.MarketCap
                
                rd = stock.FinancialStatements.IncomeStatement.ResearchAndDevelopment.TwelveMonths
                self.RD[symbol].Add(rd)
            
            # prevent storing duplicated value for the same stock in one year
            if fine_symbols.count(symbol) > 1:
                updated_flag.append(symbol)
        
        # Ability market cap weighting.
        #total_market_cap = sum([x.MarketCap for x in ability])
        #for stock, rdc in ability.items():
            #ability[stock] = rdc * (stock.MarketCap / total_market_cap)
        
        # Remove not updated symbols
        symbols_to_delete = []
        for symbol in self.RD.keys():
            if symbol not in fine_symbols:
                symbols_to_delete.append(symbol)    
        for symbol in symbols_to_delete:
            if symbol in self.RD:
                del self.RD[symbol]
        
        # starts trading after data storing period
        if len(ability) >= self.quantile:
            # Ability sorting.
            sorted_by_ability = sorted(ability.items(), key = lambda x: x[1], reverse = True)
            quantile = int(len(sorted_by_ability) / self.quantile)
            high_by_ability = [x[0].Symbol for x in sorted_by_ability[:quantile]]
            low_by_ability = [x[0].Symbol for x in sorted_by_ability[-quantile:]]
            
            self.long = high_by_ability
            self.short = low_by_ability
            #self.short = [self.technology_sector]
        
        return self.long + self.short
    
    def Selection(self):
        if self.Time.month == 4:
            self.selection_flag = True
            
    def OnData(self, data):
        if not self.selection_flag:
            return
        self.selection_flag = False
        
        # Trade execution.
        long_count = len(self.long)
        short_count = len(self.short)
        
        stocks_invested = [x.Key for x in self.Portfolio if x.Value.Invested]
        for symbol in stocks_invested:
            if symbol not in self.long + self.short:
                self.Liquidate(symbol)

        for symbol in self.long:
            if symbol in data and data[symbol]:
                self.SetHoldings(symbol, 1 / long_count)
        
        for symbol in self.short:
            if symbol in data and data[symbol]:
                self.SetHoldings(symbol, -1 / short_count)

        self.long.clear()
        self.short.clear()
        
class SymbolData():
    def __init__(self, tested_growth, period):
        self.TestedGrowth = tested_growth
        self.RD = RollingWindow[float](period)
            
    def update(self, window_value):
        self.RD.Add(window_value)
            
    def is_ready(self):
        return self.RD.IsReady
        
class CustomFeeModel(FeeModel):
    def GetOrderFee(self, parameters):
        fee = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
        return OrderFee(CashAmount(fee, "USD"))