'''
v1.5. Intersection of ROC comparison using OUT_DAY approach by Vladimir
(with dynamic stocks selector by fundamental factors and momentum)
eliminated fee saving part of the code plus daily rebalence

inspired by Peter Guenther, Tentor Testivis, Dan Whitnable, Thomas Chang, Miko M, Leandro Maia
'''
from QuantConnect.Data.UniverseSelection import *
import numpy as np
import pandas as pd
# ---------------------------------------------------------------------------------------------------------------------------
BONDS = ['TMF','TLH','SHY']; VOLA = 126; BASE_RET = 85; STK_MOM = 126; N_COARSE = 100; N_FACTOR = 20; N_MOM = 5; LEV = 1.50; HEDGE = 0.00;
# ---------------------------------------------------------------------------------------------------------------------------

class Fundamental_Factors_Momentum_ROC_Comparison_OUT_DAY(QCAlgorithm):

    def Initialize(self):

        self.SetStartDate(2008, 1, 1)  
        self.SetEndDate(2021, 1, 13)  
        self.InitCash = 100000
        self.SetCash(self.InitCash)  
        self.MKT = self.AddEquity("SPY", Resolution.Hour).Symbol
        self.mkt = []
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)
        res = Resolution.Hour      

        self.BONDS = [self.AddEquity(ticker, res).Symbol for ticker in BONDS]
        self.INI_WAIT_DAYS = 15  
        self.wait_days = self.INI_WAIT_DAYS        

        self.GLD = self.AddEquity('GLD', res).Symbol
        self.SLV = self.AddEquity('SLV', res).Symbol
        self.XLU = self.AddEquity('XLU', res).Symbol
        self.XLI = self.AddEquity('XLI', res).Symbol
        self.UUP = self.AddEquity('UUP', res).Symbol
        self.DBB = self.AddEquity('DBB', res).Symbol
        
        self.pairs = [self.GLD, self.SLV, self.XLU, self.XLI, self.UUP, self.DBB]

        self.bull = 1
        self.bull_prior = 0
        self.count = 0 
        self.outday = (-self.INI_WAIT_DAYS+1)
        self.SetWarmUp(timedelta(350))

        self.UniverseSettings.Resolution = res
        self.AddUniverse(self.CoarseFilter, self.FineFilter)
        self.data = {}
        self.RebalanceFreq = 60 
        self.UpdateFineFilter = 0
        self.symbols = None
        self.RebalanceCount = 0
        self.wt = {}
        
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.AfterMarketOpen('SPY', 30), 
            self.daily_check)
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.AfterMarketOpen('SPY', 60),
            self.trade)  
        
        symbols = [self.MKT] + self.pairs
        for symbol in symbols:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            self.SubscriptionManager.AddConsolidator(symbol, self.consolidator)
        
        self.history = self.History(symbols, VOLA, Resolution.Daily)
        if self.history.empty or 'close' not in self.history.columns:
            return
        self.history = self.history['close'].unstack(level=0).dropna()
        
        
    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-VOLA:]
        
        
    def derive_vola_waitdays(self):
        sigma = 0.6 * np.log1p(self.history[[self.MKT]].pct_change()).std() * np.sqrt(252)
        wait_days = int(sigma * BASE_RET)
        period = int((1.0 - sigma) * BASE_RET)
        return wait_days, period       
        
        
    def CoarseFilter(self, coarse):
        
        if not (((self.count-self.RebalanceCount) == self.RebalanceFreq) or (self.count == self.outday + self.wait_days - 1)):
            self.UpdateFineFilter = 0
            return Universe.Unchanged
        
        self.UpdateFineFilter = 1

        selected = [x for x in coarse if (x.HasFundamentalData) and (float(x.Price) > 5)]
        filtered = sorted(selected, key=lambda x: x.DollarVolume, reverse=True)
        return [x.Symbol for x in filtered[:N_COARSE]]
        
        
    def FineFilter(self, fundamental):
        if self.UpdateFineFilter == 0:
            return Universe.Unchanged
            
        filtered_fundamental = [x for x in fundamental if (x.ValuationRatios.EVToEBITDA > 0) 
                                        and (x.EarningReports.BasicAverageShares.ThreeMonths > 0) 
                                        and float(x.EarningReports.BasicAverageShares.ThreeMonths) * x.Price > 2e9
                                        and x.SecurityReference.IsPrimaryShare
                                        and x.SecurityReference.SecurityType == "ST00000001"
                                        and x.SecurityReference.IsDepositaryReceipt == 0
                                        and x.CompanyReference.IsLimitedPartnership == 0]

        top = sorted(filtered_fundamental, key = lambda x: x.ValuationRatios.EVToEBITDA, reverse=True)[:N_FACTOR]
        self.symbols = [x.Symbol for x in top]
        self.UpdateFineFilter = 0
        self.RebalanceCount = self.count
        return self.symbols

    
    def OnSecuritiesChanged(self, changes):
   
        addedSymbols = []
        for security in changes.AddedSecurities:
            addedSymbols.append(security.Symbol)
            if security.Symbol not in self.data:
                self.data[security.Symbol] = SymbolData(security.Symbol, STK_MOM, self)
   
        if len(addedSymbols) > 0:
            history = self.History(addedSymbols, 1 + STK_MOM, Resolution.Daily).loc[addedSymbols]
            for symbol in addedSymbols:
                try:
                    self.data[symbol].Warmup(history.loc[symbol])
                except:
                    self.Debug(str(symbol))
                    continue
        
    def calc_return(self, stocks):
        ret = {}
        for symbol in stocks:
            try:
                ret[symbol] = self.data[symbol].Roc.Current.Value
            except:
                self.Debug(str(symbol))
                continue
            
        df_ret = pd.DataFrame.from_dict(ret, orient='index')
        df_ret.columns = ['return']
        sort_return = df_ret.sort_values(by = ['return'], ascending = False)
        
        return sort_return                
 
        
    def daily_check(self):
        self.wait_days, period = self.derive_vola_waitdays()

        r = self.history.pct_change(period).iloc[-1]
        
        self.bear = ((r[self.SLV] < r[self.GLD]) and (r[self.XLI] < r[self.XLU]) and (r[self.DBB] < r[self.UUP]))
        if self.bear:
            self.bull = False
            self.outday = self.count 
        
        if (self.count >= self.outday + self.wait_days):
            self.bull = True

        self.bull_prior = self.bull
        self.count += 1

                    
    def trade(self):   
        
        if self.symbols is None: return
        output = self.calc_return(self.symbols)
        stocks = output.iloc[:N_MOM].index
        
        for sec in self.Portfolio.Keys:
            if sec not in stocks and sec not in self.BONDS:                
                self.wt[sec] = 0.
                
        for sec in stocks:
             self.wt[sec] = LEV*(1.0 - HEDGE)/len(stocks) if self.bull else LEV*HEDGE/len(stocks);
                    
        for sec in self.BONDS:
            self.wt[sec] = LEV*HEDGE/len(self.BONDS) if self.bull else LEV*(1.0 - HEDGE)/len(self.BONDS);
                
        for sec, weight in self.wt.items():
            if weight == 0. and self.Portfolio[sec].IsLong:
                self.Liquidate(sec)
                
        for sec, weight in self.wt.items(): 
            if weight != 0.:
                self.SetHoldings(sec, weight)   
    
        
    def OnEndOfDay(self): 
        
        mkt_price = self.Securities[self.MKT].Close
        self.mkt.append(mkt_price)
        mkt_perf = self.InitCash * self.mkt[-1] / self.mkt[0] 
        self.Plot('Strategy Equity', self.MKT, mkt_perf)     
        
        account_leverage = self.Portfolio.TotalHoldingsValue / self.Portfolio.TotalPortfolioValue
        
        self.Plot('Holdings', 'leverage', round(account_leverage, 2))
        self.Plot('Holdings', 'Target Leverage', LEV)
    
        
class SymbolData(object):

    def __init__(self, symbol, roc, algorithm):
        self.Symbol = symbol
        self.Roc = RateOfChange(roc)
        self.algorithm = algorithm
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, Resolution.Daily)
        algorithm.RegisterIndicator(symbol, self.Roc, self.consolidator)
        
    def Warmup(self, history):
        for index, row in history.iterrows():
            self.Roc.Update(index, row['close'])