'''
From: https://www.quantconnect.com/forum/discussion/10246/intersection-of-roc-comparison-using-out-day-approach/p1/comment-29470
Intersection of ROC comparison using OUT_DAY approach by Vladimir v1.3 
(with dynamic selector for fundamental factors and momentum)

inspired by Peter Guenther, Tentor Testivis, Dan Whitnable, Thomas Chang, Miko M, Leandro Maia

Leandro Maia setup modified by Vladimir
https://www.quantconnect.com/forum/discussion/9632/amazing-returns-superior-stock-selection-strategy-superior-in-amp-out-strategy/p2/comment-29437
'''
from QuantConnect.Data.UniverseSelection import *
import numpy as np
import pandas as pd
# --------------------------------------------------------------------------------------------------------
BONDS = ['TMF']; VOLA = 126; BASE_RET = 85; STK_MOM = 126; N_COARSE = 100; N_FACTOR = 20; N_MOM = 5; LEV = 1.00; 
# --------------------------------------------------------------------------------------------------------

class Fundamental_Factors_Momentum_ROC_Comparison_OUT_DAY(QCAlgorithm):

    def Initialize(self):

        self.SetStartDate(2008, 1, 1)  
        self.SetEndDate(2021, 1, 13)  
        self.InitCash = 1000000
        self.SetCash(self.InitCash)  
        self.MKT = self.AddEquity("SPY", Resolution.Hour).Symbol
        self.mkt = []
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)
        res = Resolution.Hour        

        self.BONDS = [self.AddEquity(ticker, res).Symbol for ticker in BONDS]
        self.INI_WAIT_DAYS = 15  
        self.wait_days = self.INI_WAIT_DAYS        

        self.GLD = self.AddEquity('GLD', res).Symbol
        self.SLV = self.AddEquity('SLV', res).Symbol
        self.XLU = self.AddEquity('XLU', res).Symbol
        self.XLI = self.AddEquity('XLI', res).Symbol
        self.UUP = self.AddEquity('UUP', res).Symbol
        self.DBB = self.AddEquity('DBB', res).Symbol
        
        self.pairs = [self.GLD, self.SLV, self.XLU, self.XLI, self.UUP, self.DBB]

        self.bull = 1
        self.bull_prior = 0
        self.count = 0 
        self.outday = (-self.INI_WAIT_DAYS+1)
        self.SetWarmUp(timedelta(350))

        self.UniverseSettings.Resolution = res
        self.AddUniverse(self.CoarseFilter, self.FineFilter)
        self.data = {}
        self.RebalanceFreq = 60 
        self.UpdateFineFilter = 0
        self.symbols = None
        self.RebalanceCount = 0
        self.wt = {}
        
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.AfterMarketOpen('SPY', 30), 
            self.daily_check)
        
        symbols = [self.MKT] + self.pairs
        for symbol in symbols:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            self.SubscriptionManager.AddConsolidator(symbol, self.consolidator)
        
        self.history = self.History(symbols, VOLA, Resolution.Daily)
        if self.history.empty or 'close' not in self.history.columns:
            return
        self.history = self.history['close'].unstack(level=0).dropna()
        
        
    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-VOLA:]
        
    def derive_vola_waitdays(self):
        sigma = 0.6 * np.log1p(self.history[[self.MKT]].pct_change()).std() * np.sqrt(252)
        wait_days = int(sigma * BASE_RET)
        period = int((1.0 - sigma) * BASE_RET)
        return wait_days, period       
        
        
    def CoarseFilter(self, coarse):
        
        if not (((self.count-self.RebalanceCount) == self.RebalanceFreq) or (self.count == self.outday + self.wait_days - 1)):
            self.UpdateFineFilter = 0
            return Universe.Unchanged
        
        self.UpdateFineFilter = 1

        selected = [x for x in coarse if (x.HasFundamentalData) and (float(x.Price) > 5)]
        filtered = sorted(selected, key=lambda x: x.DollarVolume, reverse=True)
        return [x.Symbol for x in filtered[:N_COARSE]]
        
        
    def FineFilter(self, fundamental):
        if self.UpdateFineFilter == 0:
            return Universe.Unchanged
            
        filtered_fundamental = [x for x in fundamental if (x.ValuationRatios.EVToEBITDA > 0) 
                                        and (x.EarningReports.BasicAverageShares.ThreeMonths > 0) 
                                        and float(x.EarningReports.BasicAverageShares.ThreeMonths) * x.Price > 2e9
                                        and x.SecurityReference.IsPrimaryShare
                                        and x.SecurityReference.SecurityType == "ST00000001"
                                        and x.SecurityReference.IsDepositaryReceipt == 0
                                        and x.CompanyReference.IsLimitedPartnership == 0]

        top = sorted(filtered_fundamental, key = lambda x: x.ValuationRatios.EVToEBITDA, reverse=True)[:N_FACTOR]
        self.symbols = [x.Symbol for x in top]
        self.UpdateFineFilter = 0
        self.RebalanceCount = self.count
        return self.symbols

    
    def OnSecuritiesChanged(self, changes):
   
        addedSymbols = []
        for security in changes.AddedSecurities:
            addedSymbols.append(security.Symbol)
            if security.Symbol not in self.data:
                self.data[security.Symbol] = SymbolData(security.Symbol, STK_MOM, self)
   
        if len(addedSymbols) > 0:
            history = self.History(addedSymbols, 1 + STK_MOM, Resolution.Daily).loc[addedSymbols]
            for symbol in addedSymbols:
                try:
                    self.data[symbol].Warmup(history.loc[symbol])
                except:
                    self.Debug(str(symbol))
                    continue
 
        
    def daily_check(self):
        self.wait_days, period = self.derive_vola_waitdays()

        r = self.history.pct_change(period).iloc[-1]
        
        bear = ((r[self.SLV] < r[self.GLD]) and (r[self.XLI] < r[self.XLU]) and (r[self.DBB] < r[self.UUP]))

        if bear:
            self.bull = False
            self.outday = self.count
            
        if (self.count >= self.outday + self.wait_days):
            self.bull = True
            
        self.wt_stk = LEV if self.bull else 0  
        self.wt_bnd = 0 if self.bull else LEV    
        
        if bear:
            self.trade_out()
       
        if (self.bull and not self.bull_prior) or (self.bull and (self.count==self.RebalanceCount)):
            self.trade_in()
                
        self.bull_prior = self.bull
        self.count += 1
        
        
    def trade_out(self):
        for sec in self.BONDS:
            self.wt[sec] = self.wt_bnd/len(self.BONDS)
        
        for sec in self.Portfolio.Keys:
            if sec not in self.BONDS:
                self.wt[sec] = 0
            
        for sec, weight in self.wt.items():
            if weight == 0 and self.Portfolio[sec].IsLong:
                self.Liquidate(sec)
                
        for sec, weight in self.wt.items(): 
            if weight != 0:
                self.SetHoldings(sec, weight)  


    def trade_in(self):
            
        if self.symbols is None: return
        
        output = self.calc_return(self.symbols)
        stocks = output.iloc[:N_MOM].index
        
        for sec in self.Portfolio.Keys:
            if sec not in stocks:                
                self.wt[sec] = 0
        for sec in stocks:
            self.wt[sec] = self.wt_stk/N_MOM
            
        for sec, weight in self.wt.items():             
            self.SetHoldings(sec, weight) 
 
        
    def calc_return(self, stocks):

        ret = {}
        for symbol in stocks:
            try:
                ret[symbol] = self.data[symbol].Roc.Current.Value
            except:
                self.Debug(str(symbol))
                continue
            
        df_ret = pd.DataFrame.from_dict(ret, orient='index')
        df_ret.columns = ['return']
        sort_return = df_ret.sort_values(by = ['return'], ascending = False)
        
        return sort_return
    
        
    def OnEndOfDay(self): 
        
        mkt_price = self.Securities[self.MKT].Close
        self.mkt.append(mkt_price)
        mkt_perf = self.InitCash * self.mkt[-1] / self.mkt[0] 
        self.Plot('Strategy Equity', self.MKT, mkt_perf)     
        
        account_leverage = self.Portfolio.TotalHoldingsValue / self.Portfolio.TotalPortfolioValue
        
        self.Plot('Holdings', 'leverage', round(account_leverage, 2))
        self.Plot('Holdings', 'Target Leverage', LEV)
    
        
class SymbolData(object):

    def __init__(self, symbol, roc, algorithm):
        self.Symbol = symbol
        self.Roc = RateOfChange(roc)
        self.algorithm = algorithm
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, Resolution.Daily)
        algorithm.RegisterIndicator(symbol, self.Roc, self.consolidator)
        
    def Warmup(self, history):
        for index, row in history.iterrows():
            self.Roc.Update(index, row['close'])