'''
12,321.84 %
PSR 98.875%

Intersection of ROC comparison using OUT_DAY approach by Vladimir v1.3 
(with dynamic selector for fundamental factors and momentum)

inspired by Peter Guenther, Tentor Testivis, Dan Whitnable, Thomas Chang, Miko M, Leandro Maia

Leandro Maia setup modified by Vladimir
https://www.quantconnect.com/forum/discussion/9632/amazing-returns-superior-stock-selection-strategy-superior-in-amp-out-strategy/p2/comment-29437
Changes: STK_MOM is used not only for momenum, but for average dollar volume
'''
from QuantConnect.Data.UniverseSelection import *
import numpy as np
import pandas as pd
import operator
import collections
# --------------------------------------------------------------------------------------------------------
BONDS = ['TLT']; SAFE_BONDS = ['SHY']; VOLA = 126; BASE_RET = 85; STK_MOM = 126; N_COARSE = 100; N_FACTOR = 20; N_MOM = 5; LEV = 1.00; VOLA_FCTR = 0.6;
# --------------------------------------------------------------------------------------------------------

class Fundamental_Factors_Momentum_ROC_Comparison_OUT_DAY(QCAlgorithm):

    def Initialize(self):

        # Dates and cash below changed for PROD
        self.SetStartDate(2008, 1, 1)
        #self.SetEndDate(2009, 12, 13)
       
        self.SetEndDate(2021, 3, 5)
        #self.SetEndDate(2021, 1, 13)
        #self.SetStartDate(2013, 3, 1)
        #self.SetEndDate(2013, 6, 13)
        self.InitCash = 100000
        
        
        self.SetCash(self.InitCash)  
        self.MKT = self.AddEquity("SPY", Resolution.Hour).Symbol
        self.mkt = []
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)
        res = Resolution.Hour        

        self.BONDS = [self.AddEquity(ticker, res).Symbol for ticker in BONDS]
        self.SAFE_BONDS = [self.AddEquity(ticker, res).Symbol for ticker in SAFE_BONDS]
        self.INI_WAIT_DAYS = 15  
        self.wait_days = self.INI_WAIT_DAYS        

        self.GLD = self.AddEquity('GLD', res).Symbol
        self.SLV = self.AddEquity('SLV', res).Symbol
        self.XLU = self.AddEquity('XLU', res).Symbol
        self.XLI = self.AddEquity('XLI', res).Symbol
        self.UUP = self.AddEquity('UUP', res).Symbol
        self.DBB = self.AddEquity('DBB', res).Symbol
        
        self.pairs = [self.GLD, self.SLV, self.XLU, self.XLI, self.UUP, self.DBB]

        self.bull = 1
        self.bull_prior = 0
        self.count = 0 
        self.outday = (-self.INI_WAIT_DAYS+1)
        self.SetWarmUp(timedelta(350))

        self.UniverseSettings.Resolution = res
        self.AddUniverse(self.CoarseFilter, self.FineFilter)
        self.data = {}
        self.RebalanceFreq = 60
        self.UpdateFineFilter = 0
        self.symbols = None
        self.RebalanceCount = 0
        self.wt = {}
        
        # For Avg Dollar Volume history
        self.averages = { }
        
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.AfterMarketOpen('SPY', 60),  
            self.daily_check) # change to 60 minutes back
        
        symbols = [self.MKT] + self.pairs
        for symbol in symbols:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            self.SubscriptionManager.AddConsolidator(symbol, self.consolidator)
        
        self.history = self.History(symbols, VOLA, Resolution.Daily)
        if self.history.empty or 'close' not in self.history.columns:
            return
        self.history = self.history['close'].unstack(level=0).dropna()
        
        self.correlationModel = UncorrelatedUniverseSelectionModel(windowLength = 6, historyLength = 9)
        
        
    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-VOLA:]
        
    def derive_vola_waitdays(self):
        sigma = VOLA_FCTR * np.log1p(self.history[[self.MKT]].pct_change()).std() * np.sqrt(252)
        wait_days = int(sigma * BASE_RET)
        period = int((1.0 - sigma) * BASE_RET)
        return wait_days, period       
        
    def CoarseFilter(self, coarse):
        
        if not (((self.count-self.RebalanceCount) == self.RebalanceFreq) or (self.count == self.outday + self.wait_days - 1)):
            self.UpdateFineFilter = 0
            return Universe.Unchanged
        
        self.UpdateFineFilter = 1
        
        self.correlationModel.SelectCoarse(self, coarse)

        selected = [x for x in coarse if (x.HasFundamentalData) and (float(x.Price) > 5)]#[:10]
        
        filterByDollarMomentum = True
        
        if filterByDollarMomentum:
            #https://www.quantconnect.com/terminal/index.php?key=processCache&request=embedded_backtest_9b39116ede741fb39e3eee940b4da720.html
            addedSymbols = [symbol.Symbol for symbol in selected]
            for cf in selected:
                symbol = cf.Symbol
                if cf.Symbol not in self.averages:
                    # First parameter - 21 doesn't seem to make any difference as of Feb 27, 2021
                    self.averages[cf.Symbol] = SymbolDataVolume(cf.Symbol, 21, 5)
                # Updates the SymbolData object with current EOD price - we don't need history for 5 days, result is the same
                avg = self.averages[cf.Symbol]
                avg.update(cf.EndTime, cf.AdjustedPrice, cf.DollarVolume)
            values = list(filter(lambda sd: sd.smaw.Current.Value > 0, self.averages.values()))
            
            values_str = [str(x.symbol) for x in values]
            values.sort(key=lambda x: x.smaw.Current.Value, reverse=True)
    
            # we need to return only the symbol objects
            return [ x.symbol for x in values[:N_COARSE] ]
        else:
            filtered = sorted(selected, key=lambda x: x.DollarVolume, reverse=True)
        return [x.Symbol for x in filtered[:N_COARSE]]
        
        
        
    def FineFilter(self, fundamental):
        if self.UpdateFineFilter == 0:
            return Universe.Unchanged
        
        use_custom_fine = False
        
        filtered_fundamental = [x for x in fundamental if (x.ValuationRatios.EVToEBITDA > 0)
                                        and (x.EarningReports.BasicAverageShares.ThreeMonths > 0) 
                                        and float(x.EarningReports.BasicAverageShares.ThreeMonths) * x.Price > 2e9
                                        and x.SecurityReference.IsPrimaryShare
                                        and x.SecurityReference.SecurityType == "ST00000001"
                                        and x.SecurityReference.IsDepositaryReceipt == 0
                                        and x.CompanyReference.IsLimitedPartnership == 0]
        # Doesn't make a difference
        # x.FinancialStatements.CashFlowStatement.CommonStockPayments.TwelveMonths >= 0 or <= 0
        # NetCommonStockIssuance.TwelveMonths <= 0
        # NetCommonStockIssuance.ThreeMonths <= 0
        filtered_fundamental = [x for x in filtered_fundamental if x.AssetClassification.MorningstarIndustryGroupCode != MorningstarIndustryGroupCode.DrugManufacturers]
        #filtered_fundamental = [x for x in filtered_fundamental if x.AssetClassification.MorningstarIndustryCode != MorningstarIndustryCode.IntegratedFreightAndLogistics]
        
        if use_custom_fine:
            # https://www.quantconnect.com/docs/data-library/fundamentals#Fundamentals-Reference-Tables
            # sorting: reverse=False means "longing highest",  reverse=True means "longing lowest"
            s1 = sorted(filtered_fundamental, key=lambda x: x.ValuationRatios.EVToEBITDA, reverse=False)                                  # <- added
            s2 = sorted(filtered_fundamental, key=lambda x: x.ValuationRatios.PricetoEBITDA, reverse=False)                               # <- added
            s3 = sorted(filtered_fundamental, key=lambda x: (x.ValuationRatios.PERatio if x.ValuationRatios.PERatio > 0.5 else 1000), reverse=True)                                     # <- added
    
            dict = {}
            for i, elem in enumerate(s1):                                                                                                 # <- added
                i1 = i                                                                                                                    # <- added
                i2 = s2.index(elem)                                                                                                       # <- added
                i3 = s3.index(elem)                                                                                                       # <- added
                score = sum([i1 * 0.6, i2 * 0.1, i3 * 0.3])                                                                            # <- added
                dict[elem] = score                                                                                                        # <- added
            
            top = sorted(dict.items(), key = lambda x: x[1], reverse=True)[:N_FACTOR]                                                     # <- changed
            self.symbols = [x[0].Symbol for x in top]
        else:
            top = sorted(filtered_fundamental, key = lambda x: x.ValuationRatios.EVToEBITDA, reverse=True)[:N_FACTOR]
            self.symbols = [x.Symbol for x in top]
        self.UpdateFineFilter = 0
        self.RebalanceCount = self.count
        return self.symbols

    
    def OnSecuritiesChanged(self, changes):
   
        addedSymbols = []
        for security in changes.AddedSecurities:
            addedSymbols.append(security.Symbol)
            if security.Symbol not in self.data:
                self.data[security.Symbol] = SymbolData(security.Symbol, STK_MOM, self)
   
        if len(addedSymbols) > 0:
            history = self.History(addedSymbols, 1 + STK_MOM, Resolution.Daily).loc[addedSymbols]
            for symbol in addedSymbols:
                try:
                    self.data[symbol].Warmup(history.loc[symbol])
                except:
                    self.Debug(str(symbol))
                    continue
 
        
    def daily_check(self):
        self.wait_days, period = self.derive_vola_waitdays()

        r = self.history.pct_change(period).iloc[-1]
        
        bear = ((r[self.SLV] < r[self.GLD]) and (r[self.XLI] < r[self.XLU]) and (r[self.DBB] < r[self.UUP]))

        if bear:
            self.bull = False
            self.outday = self.count
            
        if (self.count >= self.outday + self.wait_days):
            self.bull = True
            
        self.wt_stk = LEV if self.bull else 0  
        self.wt_bnd = 0 if self.bull else LEV    
        
        if bear:
            self.trade_out()
       
        if (self.bull and not self.bull_prior) or (self.bull and (self.count==self.RebalanceCount)):
            self.trade_in()
                
        self.bull_prior = self.bull
        self.count += 1
        
        
    def trade_out(self):
        try:
            sec = self.BONDS[0]
            correlationWithMKT_arr = self.correlationModel.cache[sec].correlation['A']
            correlationWithMKT = sum(correlationWithMKT_arr)/len(correlationWithMKT_arr)
        except:
            correlationWithMKT = 0
        
        bonds = self.BONDS
        if correlationWithMKT < -0.9:
            bonds = self.SAFE_BONDS
            
            #sec = self.BONDS[0]
            #correlationWithMKT = self.correlationModel.cache[sec].correlation['A'][0]
            #self.Debug('{} {} Will switch to safe bonds, correlation with SPY: {}'.format(self.Time.strftime("%m/%d/%Y %A %H:%M:%S"), str(sec), correlationWithMKT))
            
        for sec in bonds:
            self.wt[sec] = self.wt_bnd/len(bonds)
        
        for sec in self.Portfolio.Keys:
            if sec not in bonds:
                self.wt[sec] = 0
            
        for sec, weight in self.wt.items():
            if weight == 0 and self.Portfolio[sec].IsLong:
                self.Liquidate(sec)
                
        for sec, weight in self.wt.items(): 
            if weight != 0:
                self.SetHoldings(sec, weight)  
                
    def trade_out_old(self):
        for sec in self.BONDS:
            self.wt[sec] = self.wt_bnd/len(self.BONDS)
        
        for sec in self.Portfolio.Keys:
            if sec not in self.BONDS:
                self.wt[sec] = 0
            
        for sec, weight in self.wt.items():
            if weight == 0 and self.Portfolio[sec].IsLong:
                self.Liquidate(sec)
                
        for sec, weight in self.wt.items(): 
            if weight != 0:
                self.SetHoldings(sec, weight)  


    def trade_in(self):
            
        if self.symbols is None: return
        
        output = self.calc_return(self.symbols)
        stocks = output.iloc[:N_MOM].index
        
        for sec in self.Portfolio.Keys:
            if sec not in stocks:                
                self.wt[sec] = 0
        for sec in stocks:
            self.wt[sec] = self.wt_stk/N_MOM
            
        for sec, weight in self.wt.items():             
            self.SetHoldings(sec, weight) 
 
        
    def calc_return(self, stocks):

        ret = {}
        for symbol in stocks:
            try:
                ret[symbol] = self.data[symbol].Roc.Current.Value
            except:
                self.Debug(str(symbol))
                continue
            
        df_ret = pd.DataFrame.from_dict(ret, orient='index')
        df_ret.columns = ['return']
        sort_return = df_ret.sort_values(by = ['return'], ascending = False)
        
        return sort_return
    
        
    def OnEndOfDay(self): 
        
        mkt_price = self.Securities[self.MKT].Close
        self.mkt.append(mkt_price)
        mkt_perf = self.InitCash * self.mkt[-1] / self.mkt[0] 
        self.Plot('Strategy Equity', self.MKT, mkt_perf)     
        
        account_leverage = self.Portfolio.TotalHoldingsValue / self.Portfolio.TotalPortfolioValue
        
        self.Plot('Holdings', 'leverage', round(account_leverage, 2))
        self.Plot('Holdings', 'Target Leverage', LEV)
    
        
class SymbolData(object):

    def __init__(self, symbol, roc, algorithm):
        self.Symbol = symbol
        self.Roc = RateOfChange(roc)
        self.algorithm = algorithm
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, Resolution.Daily)
        algorithm.RegisterIndicator(symbol, self.Roc, self.consolidator)
        
    def Warmup(self, history):
        for index, row in history.iterrows():
            self.Roc.Update(index, row['close'])

class SelectionData():
    
    def __init__(self, history):
        self.avgDollarVolume = SimpleMovingAverage(STK_MOM)
        
        for index, row in history.iterrows():
            self.avgDollarVolume.Update(index, row['close'] * row['volume'])
        # for bar in history.itertuples():
        #     timeIndex = 1
        #     self.avgDollarVolume.Update(bar.Index[timeIndex], bar.close * bar.volume)
    
    def is_ready(self):
        return self.avgDollarVolume.IsReady
    
    def update(self, time, price, volume):
        self.avgDollarVolume.Update(time, price * volume)

class SymbolDataVolume(object):
    def __init__(self, symbol, period, periodw):
        self.symbol = symbol
        #self.tolerance = 1.01
        self.tolerance = 0.95
        self.fast = ExponentialMovingAverage(10)
        self.slow = ExponentialMovingAverage(21)
        self.is_uptrend = False
        self.scale = 0
        self.volume = 0
        self.volume_ratio = 0
        self.volume_ratiow = 0
        self.sma = SimpleMovingAverage(period)
        self.smaw = SimpleMovingAverage(periodw)

    def update(self, time, value, volume):
        self.volume = volume

        if self.smaw.Update(time, volume):
            # get ratio of this volume bar vs previous 10 before it.
            if self.smaw.Current.Value != 0:
                self.volume_ratiow = volume / self.smaw.Current.Value
        if self.sma.Update(time, volume):
            # get ratio of this volume bar vs previous 10 before it.
            if self.sma.Current.Value != 0:
                self.volume_ratio = self.smaw.Current.Value / self.sma.Current.Value
            
        if self.fast.Update(time, value) and self.slow.Update(time, value):
            fast = self.fast.Current.Value
            slow = self.slow.Current.Value
            #self.is_uptrend = fast > slow * self.tolerance
            self.is_uptrend = (fast > (slow * self.tolerance)) and (value > (fast * self.tolerance))

        if self.is_uptrend:
            self.scale = (fast - slow) / ((fast + slow) / 2.0)
            
            
            
class UncorrelatedUniverseSelectionModel:
    '''This universe selection model picks stocks that currently have their correlation to a benchmark deviated from the mean.'''

    def __init__(self,
                 benchmark = Symbol.Create("SPY", SecurityType.Equity, Market.USA),
                 tlt = Symbol.Create("TLT", SecurityType.Equity, Market.USA),
                 windowLength = 5,
                 historyLength = 25,
                 threshold = 0.5):
        '''Initializes a new default instance of the OnTheMoveUniverseSelectionModel
        Args:
            benchmark: Symbol of the benchmark
            tlt: TLT
            windowLength: Rolling window length period for correlation calculation
            historyLength: History length period
            threshold: Threadhold for the minimum mean correlation between security and benchmark'''
        

        self.benchmark = benchmark 
        self.tlt = tlt
        self.windowLength = windowLength
        self.historyLength = historyLength
        self.threshold = threshold

        self.cache = dict()
        self.symbol = list()

    def SelectCoarse(self, algorithm, coarse):
        '''Select stocks with highest Z-Score with fundamental data and positive previous-day price and volume'''

        # Verify whether the benchmark is present in the Coarse Fundamental
        benchmark = next((x for x in coarse if x.Symbol == self.benchmark), None)
        if benchmark is None:
            return self.symbol

        # Get the symbols with the highest dollar volume
        coarse = sorted([x for x in coarse if x.Symbol == self.tlt],
                        key = lambda x: x.DollarVolume, reverse=True)
        
        newSymbols = list()
        for cf in coarse + [benchmark]:
            symbol = cf.Symbol
            data = self.cache.setdefault(symbol, self.SymbolData(self, symbol))
            data.Update(cf.EndTime, cf.AdjustedPrice)
            if not data.IsReady:
                newSymbols.append(symbol)

        # Warm up the dictionary objects of selected symbols and benchmark that do not have enough data
        if len(newSymbols) > 1:
            history = algorithm.History(newSymbols, self.historyLength, Resolution.Daily)
            if not history.empty:
                history = history.close.unstack(level=0)
                for symbol in newSymbols:
                    self.cache[symbol].Warmup(history)

        # Create a new dictionary with the zScore
        zScore = dict()
        benchmark = self.cache[self.benchmark].GetReturns()
        for cf in coarse:
            symbol = cf.Symbol
            value = self.cache[symbol].CalculateZScore(benchmark)
            if value > 0: zScore[symbol] = value

        # Sort the zScore dictionary by value
        if len(zScore) > 0:
            sorted_zScore = sorted(zScore.items(), key=lambda kvp: kvp[1], reverse=True)
            zScore = dict(sorted_zScor)
        
        # Return the symbols
        self.symbols = list(zScore.keys())
        return self.symbols


    class SymbolData:
        '''Contains data specific to a symbol required by this model'''
        def __init__(self, model, symbol):
            self.symbol = symbol
            self.windowLength = model.windowLength
            self.historyLength = model.historyLength
            self.threshold = model.threshold
            self.history = RollingWindow[IndicatorDataPoint](self.historyLength)
            self.correlation = None

        def Warmup(self, history):
            '''Save the historical data that will be used to compute the correlation'''
            symbol = str(self.symbol)
            if symbol not in history:
                return

            # Save the last point before reset
            last = self.history[0]
            self.history.Reset()

            # Uptade window with historical data
            for time, value in history[symbol].iteritems():
                self.Update(time, value)

            # Re-add the last point if necessary
            if last.EndTime > time:
                self.Update(last.EndTime, last.Value)

        def Update(self, time, value):
            '''Update the historical data'''
            self.history.Add(IndicatorDataPoint(self.symbol, time, value))

        def CalculateZScore(self, benchmark):
            '''Computes the ZScore'''
            # Not enough data to compute zScore
            if not self.IsReady:
                return 0

            returns = pd.DataFrame.from_dict({"A": self.GetReturns(), "B": benchmark})

            if self.correlation is None:
                # Calculate stdev(correlation) using rolling window for all history
                correlation = returns.rolling(self.windowLength, min_periods = self.windowLength).corr()
                self.correlation = correlation["B"].dropna().unstack()
            else:
                last_correlation = returns.tail(self.windowLength).corr()["B"]
                self.correlation = self.correlation.append(last_correlation).tail(self.historyLength)

            # Calculate the mean of correlation and discard low mean correlation
            mean = self.correlation.mean()
            if mean.empty or mean[0] < self.threshold:
                return 0

            # Calculate the standard deviation of correlation
            std = self.correlation.std()

            # Current correlation
            current = self.correlation.tail(1).unstack()

            # Calculate absolute value of Z-Score for stocks in the Coarse Universe.
            return abs(current[0] - mean[0]) / std[0]

        def GetReturns(self):
            '''Get the returns from the rolling window dictionary'''
            historyDict = {x.EndTime: x.Value for x in self.history}
            return pd.Series(historyDict).sort_index().pct_change().dropna()

        @property
        def IsReady(self):
            return self.history.IsReady