from QuantConnect.Indicators import *
class ShortTermReversalVictor(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2015, 1, 1)
        self.SetEndDate(2021, 1, 1)  
        self.SetCash(100000)

        self.symbol = self.AddEquity('SPY', Resolution.Daily).Symbol
        
        self.coarse_count = 500
        self.stock_selection = 10 # 5
        self.top_by_market_cap_count = 100
        
        self.period = 21
        self.SetWarmUp(timedelta(self.period))
        
        self.long = []
        self.short = []
        
        # symbolData at each moment
        self.symbolDataDict = {}
        
        self.day = 1
        self.selection_flag = False
        self.UniverseSettings.Resolution = Resolution.Daily
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)
        self.Schedule.On(self.DateRules.EveryDay(self.symbol), self.TimeRules.AfterMarketOpen(self.symbol), self.Selection)

    def OnSecuritiesChanged(self, changes):
        # self.Debug('onChange')
        # this will get triggered initially and after each weekly rebalance
        for removed in changes.RemovedSecurities:
            self.symbolDataDict.pop(removed.Symbol, None)
        for security in changes.AddedSecurities:
            security.SetLeverage(5)
            symbol = security.Symbol
            daily_history = self.History(symbol, self.period+1, Resolution.Daily)
            if daily_history.empty:
                self.Log(f"Not enough data for {symbol} yet")
                continue
            if symbol not in self.symbolDataDict.keys():
                symbolData = SymbolData(symbol, self.period, self)
                self.symbolDataDict[symbol] = symbolData
                symbolData.warmup(daily_history)
       
    def CoarseSelectionFunction(self, coarse):
        #self.Debug('coarse')
        if not self.selection_flag: # self.selection_flag is only true when day is 5 or it is a Friday.
            return Universe.Unchanged

        selected = sorted([x for x in coarse if x.HasFundamentalData and x.Market == 'usa' and x.Price > 1],
            key=lambda x: x.DollarVolume, reverse=True)
        selected = [x.Symbol for x in selected][:self.coarse_count]

        return selected
        
    def FineSelectionFunction(self, fine): # the long and short lists are updated daily
        if not self.selection_flag: # self.selection_flag is only true when day is 5 or it is a Friday.
            return Universe.Unchanged
        fine = [x for x in fine if x.MarketCap != 0]
        
        sorted_by_market_cap = sorted(fine, key = lambda x:x.MarketCap, reverse = True)
        top_by_market_cap = [x.Symbol for x in sorted_by_market_cap[:self.top_by_market_cap_count]] # top_by_market_cap_count is 100
        
        return top_by_market_cap
    
    def OnData(self, data):
        # onData gets called after onSecuritiesChanged here because we filtered the entire data first
        #self.Debug('onData')
        for symbol in self.symbolDataDict:
            # you need to update this in case some of the stocks persisted into next selection period, which won't be warmed up in onSecuritiesChanged
            security = self.Securities[symbol]
            self.symbolDataDict[symbol].update_closes(security.Close)
            self.symbolDataDict[symbol].update_highs(security.High)
            self.symbolDataDict[symbol].update_lows(security.Low)
        if not self.selection_flag:
            return
        self.selection_flag = False
        
        """dissected_performances = {symbol : self.symbolDataDict[symbol].dissected_return() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
        sorted_by_dissected_perf = [x[0] for x in sorted(dissected_performances.items(), key=lambda item: item[1], reverse=True)]
        self.long = sorted_by_dissected_perf[::-1][:self.stock_selection]
        for symbol in sorted_by_dissected_perf: 
            if symbol not in self.long:
                self.short.append(symbol)
            if len(self.short) == self.stock_selection: # only need self.stock_selection # in short list
                break"""
        
        """monthly_capital_volume_change = {symbol : self.symbolDataDict[symbol].monthly_capital_volume_change() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
        weekly_capital_volume_change = {symbol : self.symbolDataDict[symbol].weekly_capital_volume_change() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
            
        sorted_by_month_perf = [x[0] for x in sorted(monthly_capital_volume_change.items(), key=lambda item: item[1], reverse=True)]
        sorted_by_week_perf = [x[0] for x in sorted(weekly_capital_volume_change.items(), key=lambda item: item[1])]
        self.long = sorted_by_week_perf[:self.stock_selection]
        for symbol in sorted_by_month_perf: 
            if symbol not in self.long:
                self.short.append(symbol)
            if len(self.short) == self.stock_selection: # only need self.stock_selection # in short list
                break"""
            
        """monthly_bounciness = {symbol : self.symbolDataDict[symbol].monthly_bounciness() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
        weekly_bounciness = {symbol : self.symbolDataDict[symbol].weekly_bounciness() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
            
        sorted_by_month_perf = [x[0] for x in sorted(monthly_bounciness.items(), key=lambda item: item[1], reverse=True)]
        sorted_by_week_perf = [x[0] for x in sorted(weekly_bounciness.items(), key=lambda item: item[1])]
        self.long = sorted_by_week_perf[:self.stock_selection]
        for symbol in sorted_by_month_perf: 
            if symbol not in self.long:
                self.short.append(symbol)
            if len(self.short) == self.stock_selection: # only need self.stock_selection # in short list
                break"""
            
        month_performances = {symbol : self.symbolDataDict[symbol].monthly_return() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
        week_performances = {symbol : self.symbolDataDict[symbol].weekly_return() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
            
        sorted_by_month_perf = [x[0] for x in sorted(month_performances.items(), key=lambda item: item[1], reverse=True)]
        sorted_by_week_perf = [x[0] for x in sorted(week_performances.items(), key=lambda item: item[1])]
        self.long = sorted_by_week_perf[:self.stock_selection]
        for symbol in sorted_by_month_perf: 
            if symbol not in self.long:
                self.short.append(symbol)
            if len(self.short) == self.stock_selection: # only need self.stock_selection # in short list
                break
            
        """month_performances = {symbol : self.symbolDataDict[symbol].dissected_monthly_return() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
        week_performances = {symbol : self.symbolDataDict[symbol].dissected_weekly_return() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
            
        sorted_by_month_perf = [x[0] for x in sorted(month_performances.items(), key=lambda item: item[1], reverse=True)]
        sorted_by_week_perf = [x[0] for x in sorted(week_performances.items(), key=lambda item: item[1])]
        self.long = sorted_by_month_perf[::-1][:self.stock_selection]
        for symbol in sorted_by_month_perf: 
            if symbol not in self.long:
                self.short.append(symbol)
            if len(self.short) == self.stock_selection: # only need self.stock_selection # in short list
                break"""
            
        """month_performances = {symbol : self.symbolDataDict[symbol].dissected_monthly_bounciness() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
        week_performances = {symbol : self.symbolDataDict[symbol].dissected_weekly_return() for symbol in self.symbolDataDict.keys() if self.symbolDataDict[symbol].is_ready()}
            
        sorted_by_month_perf = [x[0] for x in sorted(month_performances.items(), key=lambda item: item[1], reverse=True)]
        sorted_by_week_perf = [x[0] for x in sorted(week_performances.items(), key=lambda item: item[1])]
        self.long = sorted_by_week_perf[:self.stock_selection]
        for symbol in sorted_by_week_perf[::-1]: 
            if symbol not in self.long:
                self.short.append(symbol)
            if len(self.short) == self.stock_selection: # only need self.stock_selection # in short list
                break"""
            
        invested = [x.Key for x in self.Portfolio if x.Value.Invested]
        for symbol in invested: # if they are not to be selected again, then they are liquidated
            if symbol not in self.long:# + self.short:
                self.Liquidate(symbol)
        
        for symbol in self.long:
            if self.Securities[symbol].Price != 0 and self.Securities[symbol].IsTradable:
                self.SetHoldings(symbol, 1 / len(self.long)) # split portfolio evenly

        #for symbol in self.short:
        #    if self.Securities[symbol].Price != 0 and self.Securities[symbol].IsTradable:
        #        self.SetHoldings(symbol, -1 / len(self.short)) # split portfolio evenly
                
        self.long.clear()
        self.short.clear()

    def Selection(self):
        if self.day == 5:
            self.selection_flag = True
        
        self.day += 1
        if self.day > 5:
            self.day = 1
    

class SymbolData():
    def __init__(self, symbol, period, algo):
        self.symbol = symbol
        self.algo = algo
        self.closes = RollingWindow[float](period+1)
        self.capital_volumes = RollingWindow[float](period)
        self.highs = RollingWindow[float](period)
        self.lows = RollingWindow[float](period)
        self.period = period
        
    def update_closes(self, close):
        self.closes.Add(close)
    
    def update_highs(self, high):
        self.highs.Add(high)
    
    def update_lows(self, low):
        self.lows.Add(low)
    
    def weekly_bounciness(self):
        high = [self.highs[i] for i in range(0,5)]
        low = [self.lows[i] for i in range(self.period-5, self.period)]
        diff = sum([high[i]/low[i] - 1 for i in range(len(high))])
        return diff
    
    def monthly_bounciness(self):
        high = [self.highs[i] for i in range(self.period)]
        low = [self.lows[i] for i in range(self.period)]
        diff = sum([high[i]/low[i] - 1 for i in range(len(high))])
        return diff
        
    def dissected_weekly_bounciness(self):
        daily_return = sorted([self.highs[i]/self.lows[i] - 1 for i in range(0,6)])
        high3 = sum(daily_return[3:])
        low3 = sum(daily_return[:3])
        return high3-low3
        
    def dissected_monthly_bounciness(self) -> float:
        daily_return = sorted([self.highs[i]/self.lows[i] - 1  for i in range(0,self.period)])
        high_half = sum(daily_return[self.period//2:])
        low_half = sum(daily_return[:self.period//2])
        return high_half-low_half
    
    def update_capital_volume(self, capital_volume):
        self.capital_volumes.Add(capital_volume)
        
    def weekly_return(self) -> float:
        return self.closes[0] / self.closes[5] - 1

    def monthly_return(self) -> float:
        return self.closes[0] / self.closes[self.period-1] - 1
    
    def dissected_weekly_return(self) -> float:
        daily_return = sorted([(self.closes[i]/self.closes[i+1])-1 for i in range(0,6)])
        high3 = sum(daily_return[3:])
        low3 = sum(daily_return[:3])
        return high3-low3

    def dissected_monthly_return(self) -> float:
        daily_return = sorted([(self.closes[i]/self.closes[i+1])-1 for i in range(0,self.period)])
        high_half = sum(daily_return[self.period//2:])
        low_half = sum(daily_return[:self.period//2])
        return high_half-low_half
    
    def dissected_monthly_return(self) -> float:
        daily_return = sorted([(self.closes[i]/self.closes[i+1])-1 for i in range(0,self.period)])
        high_half = sum(daily_return[self.period//2:])
        low_half = sum(daily_return[:self.period//2])
        return high_half-low_half
        
    def is_ready(self) -> bool:
        return self.closes.IsReady and self.highs.IsReady and self.lows.IsReady
        
    def dissected_return(self):
        capital_volume = [(i,self.capital_volumes[i]) for i in range(self.period)]
        capital_volume = sorted(capital_volume, key=lambda c:c[1], reverse=True)
        daily_return = [(self.closes[i]/self.closes[i+1])-1 for i in range(0,self.period)]
        
        m_high = sum([daily_return[c[0]] for c in capital_volume[:(self.period//2)]])
        m_low = sum([daily_return[c[0]] for c in capital_volume[(self.period//2):]])
        return m_high-m_low
        
    def monthly_capital_volume_change(self) -> float:
        return self.capital_volumes[0] / self.capital_volumes[self.period-1] - 1
        
    def weekly_capital_volume_change(self) -> float:
        return self.capital_volumes[0] / self.capital_volumes[5] - 1
        
    def warmup(self, daily_history):
        if daily_history.empty:
            return
        closes = daily_history.loc[self.symbol].close
        for time, c in closes.iteritems():
            self.update_closes(c)
        highs = daily_history.loc[self.symbol].high
        for time, h in highs.iteritems():
            self.update_highs(h)
        lows = daily_history.loc[self.symbol].low
        for time, l in lows.iteritems():
            self.update_lows(l)
        capital_volumes = daily_history.loc[self.symbol].close*daily_history.loc[self.symbol].volume
        for time, capital_volume in capital_volumes.iteritems():
            self.update_capital_volume(capital_volume)

class ShortTermReversalExperimental(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2000, 1, 1)  
        self.SetEndDate(2021, 1, 1)  
        self.SetCash(100000)

        self.symbol = self.AddEquity('SPY', Resolution.Daily).Symbol
        
        self.coarse_count = 500
        self.stock_selection = 5
        self.top_by_market_cap_count = 100
        
        self.period = 21
        
        self.long = []
        self.short = []
        
        # Daily close data
        self.data = {}
        
        self.day = 1
        self.selection_flag = False
        self.UniverseSettings.Resolution = Resolution.Daily
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)
        self.Schedule.On(self.DateRules.EveryDay(self.symbol), self.TimeRules.AfterMarketOpen(self.symbol), self.Selection)

    def OnSecuritiesChanged(self, changes):
        for security in changes.AddedSecurities:
            security.SetLeverage(5)
        
    def CoarseSelectionFunction(self, coarse):
        # Update the rolling window every day, with latest prices.
        for stock in coarse:
            symbol = stock.Symbol
            if symbol not in self.Portfolio:
                self.AddEquity(symbol, Resolution.Daily)

            # Store monthly price.
            if symbol in self.data:
                self.data[symbol].update(stock.AdjustedPrice)
            
        if not self.selection_flag: # self.selection_flag is only true when day is 5 or it is a Friday.
            return Universe.Unchanged

        selected = sorted([x for x in coarse if x.HasFundamentalData and x.Market == 'usa' and x.Price > 1],
            key=lambda x: x.DollarVolume, reverse=True)
        selected = [x.Symbol for x in selected][:self.coarse_count]

        # Warmup price rolling windows.
        for symbol in selected:
            if symbol in self.data: # if data is already stored, skip
                continue

            self.data[symbol] = SymbolData(self.period) # Creates a new SymbolData object (defined below) with a period and RollingWindow of closing prices
            history = self.History(symbol, self.period, Resolution.Daily)
            if history.empty:
                self.Log(f"Not enough data for {symbol} yet")
                continue
            closes = history.loc[symbol].close
            for time, close in closes.iteritems():
                self.data[symbol].update(close)
    
        return [x for x in selected if self.data[x].is_ready()]
        
    def FineSelectionFunction(self, fine): # the long and short lists are updated daily
        fine = [x for x in fine if x.MarketCap != 0]
        
        sorted_by_market_cap = sorted(fine, key = lambda x:x.MarketCap, reverse = True)
        top_by_market_cap = [x.Symbol for x in sorted_by_market_cap[:self.top_by_market_cap_count]] # top_by_market_cap_count is 100
        
        month_performances = {symbol : self.data[symbol].monthly_return() for symbol in top_by_market_cap}
        week_performances = {symbol : self.data[symbol].weekly_return() for symbol in top_by_market_cap}
            
        sorted_by_month_perf = [x[0] for x in sorted(month_performances.items(), key=lambda item: item[1], reverse=True)]
        sorted_by_week_perf = [x[0] for x in sorted(week_performances.items(), key=lambda item: item[1])]
        
        sorted_by_week_perf = [symbol for symbol in sorted_by_week_perf if self.data[symbol].macd_diff < .01]
        # self.long = sorted_by_week_perf[:self.stock_selection] # only self.stock_selection # of these
        
        # MACD assessment component
        macd_perf = []
        for symbol in sorted_by_week_perf:
            macd = self.MACD(symbol, 12, 26, 9, MovingAverageType.Exponential, Resolution.Daily)
            if (macd.Current.Value - macd.Signal.Current.Value) / macd.Fast.Current.Value > .01:
                macd_perf.append(symbol)
        self.long = macd_perf[:self.stock_selection]
        
        for symbol in sorted_by_month_perf:
            if symbol not in self.long:
                self.short.append(symbol)
            
            if len(self.short) == self.stock_selection: # only need self.stock_selection # in short list
                break
        
        return self.long + self.short
    
    def OnData(self, data): # equities are kept for at least a week, at the end of week update portfolio
        if not self.selection_flag:
            return
        self.selection_flag = False
        
        invested = [x.Key for x in self.Portfolio if x.Value.Invested]
        for symbol in invested: # if they are not to be selected again, then they are liquidated
            if symbol not in self.long + self.short:
                self.Liquidate(symbol)
        
        for symbol in self.long:
            if self.Securities[symbol].Price != 0 and self.Securities[symbol].IsTradable:
                self.SetHoldings(symbol, 1 / len(self.long)) # split portfolio evenly

        for symbol in self.short:
            if self.Securities[symbol].Price != 0 and self.Securities[symbol].IsTradable:
                self.SetHoldings(symbol, -1 / len(self.short)) # split portfolio evenly
                
        self.long.clear()
        self.short.clear()

    def Selection(self):
        if self.day == 5:
            self.selection_flag = True
        
        self.day += 1
        if self.day > 5:
            self.day = 1