# https://quantpedia.com/strategies/short-term-reversal-with-futures/

from datetime import datetime, timedelta
from collections import deque
import numpy as np

class Short_Term_Reversal(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2005, 1, 1)                                 
        self.SetEndDate(2019, 1, 1)                               
        self.SetCash(100000) 

        self.symbols = ["CME_S1",   # Soybean Futures, Continuous Contract
                        "CME_W1",   # Wheat Futures, Continuous Contract
                        "CME_BO1",  # Soybean Oil Futures, Continuous Contract
                        "CME_C1",   # Corn Futures, Continuous Contract
                        "CME_LC1",  # Live Cattle Futures, Continuous Contract
                        "CME_FC1",  # Feeder Cattle Futures, Continuous Contract
                        "CME_GC1",  # Gold Futures, Continuous Contract
                        "CME_SI1",  # Silver Futures, Continuous Contract
                        "CME_PL1",  # Platinum Futures, Continuous Contract
                        "CME_CL1",  # Crude Oil Futures, Continuous Contract
                        
                        "ICE_CC1",  # Cocoa Futures, Continuous Contract 
                        "ICE_CT1",  # Cotton No. 2 Futures, Continuous Contract
                        "ICE_KC1",  # Coffee C Futures, Continuous Contract
                        "ICE_O1",   # Heating Oil Futures, Continuous Contract
                        "ICE_SB1",  # Sugar No. 11 Futures, Continuous Contract
                        
                        "CME_BP1", # British Pound Futures, Continuous Contract #1
                        "CME_EC1", # Euro FX Futures, Continuous Contract #1
                        "CME_JY1", # Japanese Yen Futures, Continuous Contract #1
                        "CME_SF1", # Swiss Franc Futures, Continuous Contract #1
                    
                        "CME_ES1",      # E-mini S&P 500 Futures, Continuous Contract #1
                        "CME_TY1",      # 10 Yr Note Futures, Continuous Contract #1
                        "CME_FV1",      # 5 Yr Note Futures, Continuous Contract #1


                        ]
                        
        self.lookup_period = 14
        self.SetWarmUp(self.lookup_period)
        self.close = {}
        self.volume = {}
        self.open_interest = {}
        
        # True -> Quantpedia data
        # False -> Quandl free data - NOTE: Consider shorter backtest period in case you want to use quandl data.
        self.use_quantpedia_data = True

        if self.use_quantpedia_data:
            for symbol in self.symbols:
                data = self.AddData(QuantpediaFutures, symbol, Resolution.Daily)
                #data.SetLeverage(2)
            
        self.symbols = ['CHRIS/' + x for x in self.symbols]
        
        for symbol in self.symbols:
            self.AddData(QuandlFutures, symbol, Resolution.Daily)
            
            # Open interest and volume data
            if 'CME' in symbol:
                self.AddData(QuandlFuturesOpenInterestCME, symbol, Resolution.Daily)
            else:
                self.AddData(QuandlFuturesOpenInterest, symbol, Resolution.Daily)
        
            self.AddData(QuandlFuturesVolume, symbol, Resolution.Daily)
            
            self.close[symbol] = deque(maxlen=self.lookup_period)
            self.volume[symbol] = deque(maxlen=self.lookup_period)
            self.open_interest[symbol] = deque(maxlen=self.lookup_period)
        
        # NOTE: Need to do this because of multiple quandl symbol data integration. (settle, volume, open interest)
        sym = self.symbols[0] + '.QuandlFutures'
        self.Schedule.On(self.DateRules.Every(DayOfWeek.Wednesday), self.TimeRules.AfterMarketOpen(sym), self.Rebalance)

    def OnData(self, data):
        for symbol in self.symbols:
            sym = symbol + '.QuandlFutures'
            if self.Securities.ContainsKey(sym):
                # Price update
                price = self.Securities[sym].Close
                if price != 0:
                    self.close[symbol].append(price)

            sym = symbol + '.QuandlFuturesVolume'
            if self.Securities.ContainsKey(sym):
                # Volume update
                vol = self.Securities[sym].Price
                self.volume[symbol].append(vol)
                
            # Open interest update
            open_interest_type = ''
            if 'CME' in symbol:
                open_interest_type = 'QuandlFuturesOpenInterestCME'
            else:
                open_interest_type = 'QuandlFuturesOpenInterest'
    
            sym = symbol + '.' + open_interest_type
            if self.Securities.ContainsKey(sym):
                oi = self.Securities[sym].Price
                self.open_interest[symbol].append(oi)
    
    def Rebalance(self):
        if self.IsWarmingUp: return
        
        volume_weekly_diff = {}
        oi_weekly_diff = {}
        returns = {}
        
        for symbol in self.symbols:
            if len(self.close[symbol]) == self.close[symbol].maxlen and len(self.volume[symbol]) == self.volume[symbol].maxlen and len(self.open_interest[symbol]) == self.open_interest[symbol].maxlen:
            
                # Return calc.
                prices = [x for x in self.close[symbol]]
                half = int(len(prices)/2)
                prices = prices[-half:]
                returns[symbol] = self.Return(prices)
                
                # Volume change calc.
                volumes = [x for x in self.volume[symbol]]
                
                volumes_t1 = volumes[-half:]
                t1_vol_mean = np.mean(volumes_t1)
                t1_vol_total = sum(volumes_t1) / t1_vol_mean

                volumes_t2 = volumes[:half]
                t2_vol_mean = np.mean(volumes_t2)
                t2_vol_total = sum(volumes_t2) / t2_vol_mean
                volume_weekly_diff[symbol] = t1_vol_total - t2_vol_total
                   
                # Open interest change calc.
                interests = [x for x in self.open_interest[symbol]]
                
                t1_oi = interests[-half:]
                t1_oi_total = sum(t1_oi)
                
                t2_oi = interests[:half]
                t2_oi_total = sum(t2_oi)
                oi_weekly_diff[symbol] = t1_oi_total - t2_oi_total
            #else: return

        if len(returns) == 0 or len(volume_weekly_diff) == 0 or len(oi_weekly_diff) == 0: return
        
        volume_sorted = sorted(volume_weekly_diff.items(), key = lambda x: x[1], reverse = True)
        half = int(len(volume_sorted)/2)
        high_volume = [x[0] for x in volume_sorted[:half]]
        
        open_interest_sorted = sorted(oi_weekly_diff.items(), key = lambda x: x[1], reverse = True)
        half = int(len(open_interest_sorted)/2)
        low_oi = [x[0] for x in open_interest_sorted[-half:]]
        
        filtered = [x for x in high_volume if x in low_oi]
        sorted_filtered = sorted(filtered, key = lambda x : returns[x], reverse = True)
        half = int(len(sorted_filtered)/2)
      
        long = sorted_filtered[-half:]
        short = sorted_filtered[:half]
        
        if len(long) == 0 or len(short) == 0: return
        
        # Weighting
        weight = {}
        diff = {}
        avg_ret = np.average([returns[x] for x in long+short])
        for symbol in long + short:
            diff[symbol] = np.absolute(returns[symbol] - avg_ret)
        
        if len(diff) == 0 : return
    
        total_diff = sum([x[1] for x in diff.items()])
        ratio = 1 / total_diff
        for symbol in long + short:
            weight[symbol] = diff[symbol] * ratio
        
        if len(weight) == 0 : return
    
        # Trade execution
        self.Liquidate()
        
        for symbol in long:
            sym = symbol + '.QuandlFutures'
            if self.use_quantpedia_data:
                sym = symbol.split('/')[1] + '.QuantpediaFutures'
            self.SetHoldings(sym, weight[symbol])
        for symbol in short:
            sym = symbol + '.QuandlFutures'
            if self.use_quantpedia_data:
                sym = symbol.split('/')[1] + '.QuantpediaFutures'
            self.SetHoldings(sym, -weight[symbol])
            
    def Return(self, history):
        return (history[-1] - history[0]) / history[0]

# Quantpedia data
class QuantpediaFutures(PythonData):
    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource("https://quantpedia.com/backtesting_data/futures/{0}.csv".format(config.Symbol.Value), SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)

    def Reader(self, config, line, date, isLiveMode):
        data = QuantpediaFutures()
        data.Symbol = config.Symbol
        
        try:
            if not line[0].isdigit(): return None
            split = line.split(';')
            
            data.Time = datetime.strptime(split[0], "%d.%m.%Y") + timedelta(days=1)
            data['settle'] = float(split[1])
            data.Value = float(split[1])
        except:
            return None
            
        return data

# Quandl free data
class QuandlFutures(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "Settle"


class QuandlFuturesOpenInterest(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "Prev. Day Open Interest"

class QuandlFuturesOpenInterestCME(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "Previous Day Open Interest"
        

class QuandlFuturesVolume(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "Volume"