'''
1.3
Intersection of ROC comparison using OUT_DAY approach by Vladimir v1.3 
(with dynamic selector for fundamental factors and momentum)

inspired by Peter Guenther, Tentor Testivis, Dan Whitnable, Thomas Chang, Miko M, Leandro Maia

Leandro Maia setup modified by Vladimir
https://www.quantconnect.com/forum/discussion/9632/amazing-returns-superior-stock-selection-strategy-superior-in-amp-out-strategy/p2/comment-29437
'''
from QuantConnect.Data.UniverseSelection import *
import numpy as np
import pandas as pd
# --------------------------------------------------------------------------------------------------------
BONDS = ['TLT']; VOLA = 126; BASE_RET = 85; STK_MOM = 126; N_COARSE = 1000; N_FACTOR = 100; N_MOM = 5; LEV = 1.00; 
# --------------------------------------------------------------------------------------------------------

class Fundamental_Factors_Momentum_ROC_Comparison_OUT_DAY(QCAlgorithm):

    def Initialize(self):

        self.SetStartDate(2008, 1, 1)  
        self.SetEndDate(2021, 2, 1)  
        self.InitCash = 100000
        self.SetCash(self.InitCash)  
        self.MKT = self.AddEquity("SPY", Resolution.Hour).Symbol
        self.mkt = []
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)
        res = Resolution.Hour        

        self.BONDS = [self.AddEquity(ticker, res).Symbol for ticker in BONDS]
        self.INI_WAIT_DAYS = 15  
        self.wait_days = self.INI_WAIT_DAYS        

        self.GLD = self.AddEquity('GLD', res).Symbol
        self.SLV = self.AddEquity('SLV', res).Symbol
        self.XLU = self.AddEquity('XLU', res).Symbol
        self.XLI = self.AddEquity('XLI', res).Symbol
        self.UUP = self.AddEquity('UUP', res).Symbol
        self.DBB = self.AddEquity('DBB', res).Symbol
        
        self.pairs = [self.GLD, self.SLV, self.XLU, self.XLI, self.UUP, self.DBB]

        self.bull = 1
        self.bull_prior = 0
        self.count = 0 
        self.outday = (-self.INI_WAIT_DAYS+1)
        self.SetWarmUp(timedelta(350))

        self.UniverseSettings.Resolution = res
        self.AddUniverse(self.CoarseFilter, self.FineFilter)
        self.data = {}
        self.RebalanceFreq = 60 
        self.UpdateFineFilter = 0
        self.symbols = None
        self.RebalanceCount = 0
        self.wt = {}
        
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.AfterMarketOpen('SPY', 30), 
            self.daily_check)
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.AfterMarketOpen('SPY', 60),
            self.trade)  
        
        symbols = [self.MKT] + self.pairs
        for symbol in symbols:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            self.SubscriptionManager.AddConsolidator(symbol, self.consolidator)
        
        self.history = self.History(symbols, VOLA, Resolution.Daily)
        if self.history.empty or 'close' not in self.history.columns:
            return
        self.history = self.history['close'].unstack(level=0).dropna()
        
        
    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-VOLA:]
        
    def derive_vola_waitdays(self):
        sigma = 0.6 * np.log1p(self.history[[self.MKT]].pct_change()).std() * np.sqrt(252)
        wait_days = int(sigma * BASE_RET)
        period = int((1.0 - sigma) * BASE_RET)
        return wait_days, period       
        
        
    def CoarseFilter(self, coarse):
        
        if not (((self.count-self.RebalanceCount) == self.RebalanceFreq) or (self.count == self.outday + self.wait_days - 1)):
            self.UpdateFineFilter = 0
            return Universe.Unchanged
        
        self.UpdateFineFilter = 1

        selected = [x for x in coarse if (x.HasFundamentalData) and (float(x.Price) > 3)]
        filtered = sorted(selected, key=lambda x: x.DollarVolume, reverse=True)
        return [x.Symbol for x in filtered[:N_COARSE]]
        
        
    def FineFilter(self, fundamental):
        if self.UpdateFineFilter == 0:
            return Universe.Unchanged
        
        universe_valid = [x for x in fundamental
            if float(x.EarningReports.BasicAverageShares.ThreeMonths) * x.Price > 1e9
            and x.SecurityReference.IsPrimaryShare
            and x.SecurityReference.SecurityType == "ST00000001"
            and x.SecurityReference.IsDepositaryReceipt == 0
            and x.CompanyReference.IsLimitedPartnership == 0
            and x.OperationRatios.ROIC
            and x.OperationRatios.CapExGrowth
            and x.OperationRatios.FCFGrowth
            and x.ValuationRatios.BookValueYield
            and x.ValuationRatios.EVToEBITDA
            and x.ValuationRatios.PricetoEBITDA
            and x.ValuationRatios.PERatio
            and x.ValuationRatios.FCFYield
            ]
        
        returns, volatility, sharpe_ratio = self.get_momentum(universe_valid)
        
        sortedByfactor0 = sorted(universe_valid, key=lambda x: returns[x.Symbol],                        reverse=False) # high return or sharpe or low volatility
        sortedByfactor1 = sorted(universe_valid, key=lambda x: x.OperationRatios.ROIC.OneYear,           reverse=False) # high ROIC
        sortedByfactor2 = sorted(universe_valid, key=lambda x: x.OperationRatios.CapExGrowth.ThreeYears, reverse=False) # high growth
        sortedByfactor3 = sorted(universe_valid, key=lambda x: x.OperationRatios.FCFGrowth.ThreeYears,   reverse=False) # high growth
        sortedByfactor4 = sorted(universe_valid, key=lambda x: x.ValuationRatios.BookValueYield,         reverse=False) # high Book Value Yield
        sortedByfactor5 = sorted(universe_valid, key=lambda x: x.ValuationRatios.EVToEBITDA,             reverse=True)  # low enterprise value to EBITDA
        sortedByfactor6 = sorted(universe_valid, key=lambda x: x.ValuationRatios.PricetoEBITDA,          reverse=True)  # low share price to EBITDA
        sortedByfactor7 = sorted(universe_valid, key=lambda x: x.ValuationRatios.PERatio,                reverse=True)  # low share price to its per-share earnings
        sortedByfactor8 = sorted(universe_valid, key=lambda x: x.ValuationRatios.FCFYield,               reverse=False)

        stock_dict = {}
        for i, elem in enumerate(sortedByfactor0):
            rank0 = i
            rank1 = sortedByfactor1.index(elem)
            rank2 = sortedByfactor2.index(elem)
            rank3 = sortedByfactor3.index(elem)
            rank4 = sortedByfactor4.index(elem)
            rank5 = sortedByfactor5.index(elem)
            rank6 = sortedByfactor6.index(elem)
            rank7 = sortedByfactor7.index(elem)
            rank8 = sortedByfactor8.index(elem)
            score = sum([rank0*1.0, rank1*0.0, rank2*0.0, rank3*0.0, rank4*0.0, rank5*0.0, rank6*0.0, rank7*0.0, rank8*1.0])
            stock_dict[elem] = score
        
        self.sorted_stock_dict = sorted(stock_dict.items(), key=lambda x:x[1], reverse=True)
        sorted_symbol = [x[0] for x in self.sorted_stock_dict]
        
        top  = [x for x in sorted_symbol[:N_FACTOR]]
        self.symbols = [i.Symbol for i in top]
        self.UpdateFineFilter = 0
        self.RebalanceCount = self.count
        return self.symbols
        
        
    def get_momentum(self, universe):
        symbols = [i.Symbol for i in universe]
        hist_df = self.History(symbols, 63, Resolution.Daily)
        returns = {}
        volatility = {}
        sharpe = {}
        for s in symbols:
            ret = np.log( hist_df.loc[str(s)]['close'] / hist_df.loc[str(s)]['close'].shift(1) )
            returns[s] = ret.mean() * 252
            volatility[s] = ret.std() * np.sqrt(252) 
            sharpe[s] = (returns[s] - 0.03) / volatility[s]
        return returns, volatility, sharpe
    
    def OnSecuritiesChanged(self, changes):
   
        addedSymbols = []
        for security in changes.AddedSecurities:
            addedSymbols.append(security.Symbol)
            if security.Symbol not in self.data:
                self.data[security.Symbol] = SymbolData(security.Symbol, STK_MOM, self)
   
        if len(addedSymbols) > 0:
            history = self.History(addedSymbols, 1 + STK_MOM, Resolution.Daily).loc[addedSymbols]
            for symbol in addedSymbols:
                try:
                    self.data[symbol].Warmup(history.loc[symbol])
                except:
                    self.Debug(str(symbol))
                    continue
 
    def calc_return(self, stocks):
        ret = {}
        for symbol in stocks:
            try:
                ret[symbol] = self.data[symbol].Roc.Current.Value
            except:
                self.Debug(str(symbol))
                continue
            
        df_ret = pd.DataFrame.from_dict(ret, orient='index')
        df_ret.columns = ['return']
        sort_return = df_ret.sort_values(by = ['return'], ascending = False)
        
        return sort_return

    def daily_check(self):
        self.wait_days, period = self.derive_vola_waitdays()

        r = self.history.pct_change(period).iloc[-1]
        
        self.bear = ((r[self.SLV] < r[self.GLD]) and (r[self.XLI] < r[self.XLU]) and (r[self.DBB] < r[self.UUP]))

        if self.bear:
            self.bull = False
            self.outday = self.count
            
        if (self.count >= self.outday + self.wait_days):
            self.bull = True
            
        self.wt_stk = LEV if self.bull else 0  
        self.wt_bnd = 0 if self.bull else LEV    


    def trade(self):
        if self.bear:
            self.trade_out()
       
        if (self.bull and not self.bull_prior) or (self.bull and (self.count==self.RebalanceCount)):
            self.trade_in()

        self.bull_prior = self.bull
        self.count += 1            
        
        
    def trade_out(self):
        for sec in self.BONDS:
            self.wt[sec] = self.wt_bnd/len(self.BONDS)
        
        for sec in self.Portfolio.Keys:
            if sec not in self.BONDS:
                self.wt[sec] = 0
            
        for sec, weight in self.wt.items():
            if weight == 0 and self.Portfolio[sec].IsLong:
                self.Liquidate(sec)
                
        for sec, weight in self.wt.items(): 
            if weight != 0:
                self.SetHoldings(sec, weight)  


    def trade_in(self):
            
        if self.symbols is None: return
        
        output = self.calc_return(self.symbols)
        stocks = output.iloc[:N_MOM].index
        
        for sec in self.Portfolio.Keys:
            if sec not in stocks:                
                self.wt[sec] = 0
        for sec in stocks:
            self.wt[sec] = self.wt_stk/N_MOM
            
        for sec, weight in self.wt.items():             
            self.SetHoldings(sec, weight) 
    
        
    def OnEndOfDay(self): 
        
        mkt_price = self.Securities[self.MKT].Close
        self.mkt.append(mkt_price)
        mkt_perf = self.InitCash * self.mkt[-1] / self.mkt[0] 
        self.Plot('Strategy Equity', self.MKT, mkt_perf)     
        
        account_leverage = self.Portfolio.TotalHoldingsValue / self.Portfolio.TotalPortfolioValue
        
        self.Plot('Holdings', 'leverage', round(account_leverage, 2))
        self.Plot('Holdings', 'Target Leverage', LEV)
    
        
class SymbolData(object):

    def __init__(self, symbol, roc, algorithm):
        self.Symbol = symbol
        self.Roc = RateOfChange(roc)
        self.algorithm = algorithm
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, Resolution.Daily)
        algorithm.RegisterIndicator(symbol, self.Roc, self.consolidator)
        
    def Warmup(self, history):
        for index, row in history.iterrows():
            self.Roc.Update(index, row['close'])