# https://quantpedia.com/strategies/short-term-reversal-with-futures/
#
# The investment universe consists of 24 types of US futures contracts (4 currencies, 5 financials, 8 agricultural, 7 commodities). 
# A weekly time frame is used – a Wednesday- Wednesday interval. The contract closest to expiration is used, except within the delivery 
# month, in which the second-nearest contract is used. Rolling into the second nearest contract is done at the beginning of the delivery month.
# The contract is defined as the high- (low-) volume contract if the contract’s volume changes between period from t-1 to t and period from t-2
# to t-1 is above (below) the median volume change of all contracts (weekly trading volume is detrended by dividing the trading volume by its 
# sample mean to make the volume measure comparable across markets). All contracts are also assigned to either high-open interest (top 50% of 
# changes in open interest) or low-open interest groups (bottom 50% of changes in open interest) based on lagged changes in open interest between
# the period from t-1 to t and period from t-2 to t-1. The investor goes long (short) on futures from the high-volume, low-open interest group 
# with the lowest (greatest) returns in the previous week. The weight of each contract is proportional to the difference between the return
# of the contract over the past one week and the equal-weighted average of returns on the N (number of contracts in a group) contracts during that period.

from collections import deque
import numpy as np

class ShortTermReversal(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2000, 1, 1) 
        self.SetCash(100000) 

        self.symbols = [
                        "CME_S1",   # Soybean Futures, Continuous Contract
                        "CME_W1",   # Wheat Futures, Continuous Contract
                        "CME_BO1",  # Soybean Oil Futures, Continuous Contract
                        "CME_C1",   # Corn Futures, Continuous Contract
                        "CME_LC1",  # Live Cattle Futures, Continuous Contract
                        "CME_FC1",  # Feeder Cattle Futures, Continuous Contract
                        "CME_GC1",  # Gold Futures, Continuous Contract
                        "CME_SI1",  # Silver Futures, Continuous Contract
                        "CME_PL1",  # Platinum Futures, Continuous Contract
                        "CME_CL1",  # Crude Oil Futures, Continuous Contract

                        "ICE_RS1",  # Canola Futures, Continuous Contract
                        "ICE_GO1",  # Gas Oil Futures, Continuous Contract
                        "CME_RB2",  # Gasoline Futures, Continuous Contract
                        "CME_KW2",  # Wheat Kansas, Continuous Contract
                        "ICE_WT1",  # WTI Crude Futures, Continuous Contract

                        "ICE_CC1",  # Cocoa Futures, Continuous Contract 
                        "ICE_CT1",  # Cotton No. 2 Futures, Continuous Contract
                        "ICE_KC1",  # Coffee C Futures, Continuous Contract
                        "ICE_O1",   # Heating Oil Futures, Continuous Contract
                        "ICE_SB1",  # Sugar No. 11 Futures, Continuous Contract
                        
                        "CME_BP1", # British Pound Futures, Continuous Contract #1
                        "CME_EC1", # Euro FX Futures, Continuous Contract #1
                        "CME_JY1", # Japanese Yen Futures, Continuous Contract #1
                        "CME_SF1", # Swiss Franc Futures, Continuous Contract #1
                    
                        "CME_ES1",  # E-mini S&P 500 Futures, Continuous Contract #1
                        "CME_TY1",  # 10 Yr Note Futures, Continuous Contract #1
                        "CME_FV1",  # 5 Yr Note Futures, Continuous Contract #1
                        ]
                        
        self.period = 14
        self.SetWarmUp(self.period)
        
        # Daily close, volume and open interest data.
        self.data = {}
        self.rebalance_flag = False
        
        # Price data.
        for symbol in self.symbols:
            data = self.AddData(QuantpediaFutures, symbol, Resolution.Daily)
            data.SetFeeModel(CustomFeeModel(self))
            
            self.data[symbol] = deque(maxlen=self.period)
            
            # Open interest and volume data
            symbol = 'CHRIS/' + symbol
            if 'CME' in symbol:
                self.AddData(QuandlFuturesOpenInterestCME, symbol, Resolution.Daily)
            else:
                self.AddData(QuandlFuturesOpenInterest, symbol, Resolution.Daily)
            self.AddData(QuandlFuturesVolume, symbol, Resolution.Daily)

        # NOTE: Need to do this because of multiple symbol data integration. (settle, volume, open interest)
        sym = self.symbols[0] + '.QuantpediaFutures'
        self.Schedule.On(self.DateRules.Every(DayOfWeek.Wednesday), self.TimeRules.AfterMarketOpen(sym), self.Rebalance)

    def OnData(self, data):
        for symbol in self.symbols:

            volume_symbol = 'CHRIS/' + symbol + '.QuandlFuturesVolume'

            # Open interest update
            open_interest_type = ''
            if 'CME' in symbol:
                open_interest_type = 'QuandlFuturesOpenInterestCME'
            else:
                open_interest_type = 'QuandlFuturesOpenInterest'
            open_interest_symbol = 'CHRIS/' + symbol + '.' + open_interest_type
    
            # if self.Securities.ContainsKey(symbol) and self.Securities.ContainsKey(volume_symbol) and self.Securities.ContainsKey(open_interest_symbol):
            if symbol in data and volume_symbol in data and open_interest_symbol in data:
                if data[symbol] and data[volume_symbol] and data[open_interest_symbol]:
                    price = data[symbol].Value
                    vol = data[volume_symbol].Value
                    oi = data[open_interest_symbol].Value
                
                    if price != 0 and vol != 0 and oi != 0:
                        self.data[symbol].append((price, vol, oi))

    def Rebalance(self):
        if self.IsWarmingUp: return
        
        ret_volume_oi_data = {}
        for symbol in self.symbols:
            # Data is ready.
            if len(self.data[symbol]) == self.data[symbol].maxlen:
            
                # Return calc.
                prices = [x[0] for x in self.data[symbol]]
                half = int(len(prices)/2)
                prices = prices[-half:]
                ret = prices[-1] / prices[0] - 1
                
                # Volume change calc.
                volumes = [x[1] for x in self.data[symbol]]
                
                volumes_t1 = volumes[-half:]
                t1_vol_mean = np.mean(volumes_t1)
                t1_vol_total = sum(volumes_t1) / t1_vol_mean

                volumes_t2 = volumes[:half]
                t2_vol_mean = np.mean(volumes_t2)
                t2_vol_total = sum(volumes_t2) / t2_vol_mean
                volume_weekly_diff = t1_vol_total - t2_vol_total
                   
                # Open interest change calc.
                interests = [x[2] for x in self.data[symbol]]
                
                t1_oi = interests[-half:]
                t1_oi_total = sum(t1_oi)
                
                t2_oi = interests[:half]
                t2_oi_total = sum(t2_oi)
                oi_weekly_diff = t1_oi_total - t2_oi_total
                
                # Store weekly diff data.
                ret_volume_oi_data[symbol] = (ret, volume_weekly_diff, oi_weekly_diff)

        long = []
        short = []
        if len(ret_volume_oi_data) != 0:
            volume_sorted = sorted(ret_volume_oi_data.items(), key = lambda x: x[1][1], reverse = True)
            half = int(len(volume_sorted)/2)
            high_volume = [x for x in volume_sorted[:half]]
            
            open_interest_sorted = sorted(ret_volume_oi_data.items(), key = lambda x: x[1][2], reverse = True)
            half = int(len(open_interest_sorted)/2)
            low_oi = [x for x in open_interest_sorted[-half:]]
            
            filtered = [x for x in high_volume if x in low_oi]
            filtered_by_return = sorted(filtered, key = lambda x : x[0], reverse = True)
            half = int(len(filtered_by_return) / 2)
          
            long = filtered_by_return[-half:]
            short = filtered_by_return[:half]
        
        # Make sure we have at least two values for weighting.
        if len(long + short) < 2: return
        
        # Return weighting.
        weight = {}
        diff = {}
        avg_ret = np.average([x[1][0] for x in long + short])
        
        for symbol, ret_volume_oi in long + short:
            diff[symbol] = ret_volume_oi[0] - avg_ret
        
        total_diff = sum([abs(x[1]) for x in diff.items()])
        long_symbols = [x[0] for x in long]
        
        if total_diff == 0: return

        for symbol, data in long + short:
            if symbol in long_symbols:
                weight[symbol] = diff[symbol] / total_diff
            else:
                weight[symbol] = - diff[symbol] / total_diff
        
        # Trade execution
        self.Liquidate()
        
        for symbol, w in weight.items():
            sym = symbol + '.QuantpediaFutures'
            self.SetHoldings(sym, w)

class QuandlFuturesOpenInterest(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "Prev. Day Open Interest"

class QuandlFuturesOpenInterestCME(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "Previous Day Open Interest"
        
class QuandlFuturesVolume(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "Volume"

# Quantpedia data.
# NOTE: IMPORTANT: Data order must be ascending (datewise)
class QuantpediaFutures(PythonData):
    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource("data.quantpedia.com/backtesting_data/futures/{0}.csv".format(config.Symbol.Value), SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)

    def Reader(self, config, line, date, isLiveMode):
        data = QuantpediaFutures()
        data.Symbol = config.Symbol
        
        if not line[0].isdigit(): return None
        split = line.split(';')
        
        data.Time = datetime.strptime(split[0], "%d.%m.%Y") + timedelta(days=1)
        data['back_adjusted'] = float(split[1])
        data['spliced'] = float(split[2])
        data.Value = float(split[1])

        return data        

# Custom fee model.
class CustomFeeModel(FeeModel):
    def GetOrderFee(self, parameters):
        fee = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
        return OrderFee(CashAmount(fee, "USD"))