Overall Statistics
Total Trades
3892
Average Win
0.27%
Average Loss
-0.26%
Compounding Annual Return
6.054%
Drawdown
1.500%
Expectancy
0.082
Net Profit
57.545%
Sharpe Ratio
0.495
Probabilistic Sharpe Ratio
0.000%
Loss Rate
47%
Win Rate
53%
Profit-Loss Ratio
1.05
Alpha
0.045
Beta
0.001
Annual Standard Deviation
0.091
Annual Variance
0.008
Information Ratio
-0.194
Tracking Error
0.177
Treynor Ratio
34.959
Total Fees
$1109662.96
Estimated Strategy Capacity
$580000.00
Lowest Capacity Asset
CZM UI5APSUZZHWL
#region imports
from AlgorithmImports import *
#endregion
from Information import *
from UniverseHelpers import LoadSymbolData

#### HERE IS WHERE THE MANUAL UNIVERSE SELECTION TICKES ARE DEFINED FOR ALL INTENTS AND PURPOSES #######################
# Append a defined dictionary to add to to the UniverseSelectionModel later on.

# LETFInformation is the NAME OF IMPORTED OBJECT that will be used directly in the algorithm. See comments for LoadSymbolData 

MajorIndicies = [Russell, NASDAQ, SP500,DowJones]
NotTheUS= [Russia,DevelopedMSCI,China,Japan]
Commodities = [Miners, JuniorMiners,Gold,BloombergSilverIndex]
DowSectorSpecific = [DowMaterials,Biotech,DowFinancials,DowHealth, DowIndustrials,DowOilGas, DowUtilities]
SPSector = [SP500SmallCap,SP500MidCap, SP500OilGas,SP500Energy, SP500Tech]
Currencies = [YenUSD]

Working = [NASDAQ, DowJones, Russell, Russia, YenUSD,DowJones]


BACKTESTED_SUBUNIVERSES = [NASDAQ, Russell,Miners]+Commodities+Currencies+SPSector+NotTheUS
DirectionalTrading = False
VolTrading = True
# See comments in Information.py lines 12-23 for what these objects are. They are imported into main.
LETFInformation, PairsList = LoadSymbolData(BACKTESTED_SUBUNIVERSES)

BarSize =1 #Minutes - How frequently to look to make orders.
TradingFrequency = 60*6
#NoiseFilter is the abs minumum value of the Spread we must overreach before we consider it an Insight.

asPair = False
LS = 1
AntiSpread = -1
RollingWindowLength = 5000 # 1 Trading day in Minutes.
#DiscountSpreadThreshold= -.0050
PremiumSpreadThreshold = .0015
DiscountSpreadThreshold= -.0015
#PremiumSpreadThreshold = 3
WonkSpread = .00025
FixDollarSize = .02
TakeProfit= 0.02 #centage at which to start to liquidate regardless of Spread
BetterExecution = -.0001
beta_ = (False, 2/11)
RebarrelThreshold = -.2
#region imports
from AlgorithmImports import *
#endregion
NASDAQ = {
    "Benchmark": "QQQ",
    "BullETFs": 
        {  "TQQQ":3,
            "QLD":2,
            "QQQ":1
        },
    "BearETFs": {
            "SQQQ":-3,
            "QID":-2, 
            "PSQ":-1
            
            },
    "Trade":{
        "TQQQ":"SQQQ",
        "QLD":"QID",
        #"QQQ":"PSQ"
        
    }
}

SP500 = {
    "Benchmark": "SPY",
    "BullETFs":  { 
            "UPRO":3,
            "SDS":2,
            "SPY":1
        
    },
    "BearETFs": 
            {
            "SPXU":-3,
            "SSO":-2 ,
            "SH":-1
            },
    "Trade":{
        "UPRO":"SPXU",
        #"SDS":"SSO",
        #"SPY":"SH"
    }
    
}
   
Russell = {
    "Benchmark": "IWM",
    "BullETFs": 
        {  "TNA":3,
            "URTY":3,
            "UWM":2,
            "IWM":1
        },
    "BearETFs": {
            "TZA":-3 ,
            "SRTY":-3,
            "TWM":-2 ,
            "RWM":-1
            },
    "Trade":{
        #"TNA":"TZA",
        "URTY":"SRTY",
        #"UWM":"TWM",
        #"IWM":"RWM"
    }
}         

DowJones = {
    "Benchmark": "DIA",
    "BullETFs": 
        {  
            "UDOW":3,
            "DDM":2,
            "DIA":1,
        },
    "BearETFs": {
           "SDOW":-3,
            "DXD":-2,
            "DOG":-1,
            },
    "Trade":{
        "UDOW":"SDOW",
        #"DDM":"DXD",
        #"DIA":"DOG",
        
    }
}

Russia= {
    "Benchmark": "RSX",
    "BullETFs":  
        { 
            "RUSL":3
        },
    "BearETFs": 
            {
            "RUSS":-3
            }
            }
            
DevelopedMSCI = {
    "Benchmark": "EFA",
    "BullETFs":  
        { 
            "EFO":2
        },
    "BearETFs": 
            {
            "EFU":-2
            }
            }   

China = {
    "Benchmark": "FXI",
    "BullETFs":  
        { 
            "YINN":3,
            "XXP":2
        },
    "BearETFs": 
            {
            "YANG":-3,
            "FXP":-2,
            "YXI":-1
            },
    "Trade":{
        "YINN":"YANG",
        #"XXP":"FXP",
        #"FXI":"YXI"
    }
            }

Japan= { 
    "Benchmark": "EWJ",
    "BullETFs":  
        { 
            "EZJ":2
        },
    "BearETFs": 
            {
            "EWV":-2
            }
            }

Miners = {
     "Benchmark": "GDX",
    "BullETFs":  
        { 
            "NUGT":2
        },
    "BearETFs": 
            {
            "DUST":-2
            }
            }

JuniorMiners = {
     "Benchmark": "GDXJ",
    "BullETFs":  
        { 
            "JNUG":2
        },
    "BearETFs": 
            {
            "JDST":-2
            }
            }

Gold = {
     "Benchmark": "GLD",
    "BullETFs":  
        { 
            "UGL":2
        },
    "BearETFs": 
            {
            "GLL":-2
            }
            }

DowMaterials = {
     "Benchmark": "IYM",
    "BullETFs":  
        { 
            "UYM":2
        },
    "BearETFs": 
            {
            "SBM":-2
            }
            }

Biotech = {
     "Benchmark": "IBB",
    "BullETFs":  
        { 
            "BIB":2
        },
    "BearETFs": 
            {
            "BIS":-2
            }
            }

DowFinancials = {
    #SEF is the -1x and Bull/Bear defined relative to benchmark
      "Benchmark": "SEF",
    "BullETFs":  
        { 
            "SKF":2 # actually -2x the index
        },
    "BearETFs": 
            {
            "UYG":-2   # actually 2x the index
            }
}

DowHealth = {
      "Benchmark": "IYH",
    "BullETFs":  
        { 
            "RXL":2
        },
    "BearETFs": 
            {
            "RXD":-2   
            }
}



DowIndustrials = {
      "Benchmark": "IYJ",
    "BullETFs":  
        { 
            "UXI":2 
        },
    "BearETFs": 
            {
            "SIJ":-2 
            }
}

DowOilGas = {
      "Benchmark": "IYE",
    "BullETFs":  
        { 
            "DIG":2,
            "IYE":1
        },
    "BearETFs": 
            {
            "DUG":-2,  
            "DDG":-1},
    "Trade":{
        "DIG":"DUG",
        "IYE":"DDG"
    }
}

DowRealEstate = {
      "Benchmark": "IYR",
    "BullETFs":  
        { 
            "URE":2,
            "IYR":1
        },
    "BearETFs": 
            {
            "SRS":-2,  
            "REK":-1},
    "Trade":{
        "URE":"SRS",
        "IYR":"REK"}
}

DowUtilities = {
      "Benchmark": "IDU",
    "BullETFs":  
        { 
            "UPW":2 
        },
    "BearETFs": 
            {
            "SDP":-2 
            }
}

SP500SmallCap = {
      "Benchmark": "IJR",
    "BullETFs":  
        { 
            "SAA":2,
            "IJR":1
        },
    "BearETFs": 
            {
            "SDD":-2, 
            "SBB":-1 
            },
    "Trade":
        {
            "SAA":"SDD",
            "IJR":"SBB"
        }
}

SP500MidCap = {
      "Benchmark": "IJH",
    "BullETFs":  
        { 
            "UMDD":3,
            "MVV":2,
            "IJH":1
        },
    "BearETFs": 
            {
            "SMDD":-3,
            "MZZ": -2,
            "SBB":-1, 
            },
    "Trade": {
        "UMDD":"SMDD",
        "MVV":"MZZ",
        "IJH":"SBB",
        
        
        
    }
}

BloombergSilverIndex =  {
      "Benchmark": "SLV",
    "BullETFs":  
        { 
            "AGQ":2 
        },
    "BearETFs": 
            {
            "ZSL":-2 
            }
}

YenUSD =  {
      "Benchmark": "FXY",
    "BullETFs":  
        { 
            "YCS":2 
        },
    "BearETFs": 
            {
            "YCL":-2 
            }
}

SP500OilGas =  {
      "Benchmark": "XOP",
    "BullETFs":  
        { 
            "GUSH":2 
        },
    "BearETFs": 
            {
            "DRIP":-2 
            }
}

SP500Energy =  {
      "Benchmark": "XLE",
    "BullETFs":  
        { 
            "ERX":2 
        },
    "BearETFs": 
            {
            "ERY":-2 
            }
}


SP500Tech =  {
      "Benchmark": "XLK",
    "BullETFs":  
        { 
            "TECL":2 
        },
    "BearETFs": 
            {
            "TECS":-2 
            }
}
USTreasury = {
      "Benchmark": "TLT",
    "BullETFs":  
        { 
            "TMF":3,  #Direxion
            "UBT":2 
        },
    "BearETFs": 
            {
            "TMV": -3 , #Direxion
            "TBT":-2,
            "TBF": -1
            },
    "Trade": {
            "TMF":"TMV",
            "UBT":"TBT",
            "TLT": "TBF"
    }
}
#region imports
from AlgorithmImports import *
#endregion
pass
#region imports
from AlgorithmImports import *
#endregion
import Configure as config
from clr import AddReference
AddReference("System")
AddReference("QuantConnect.Common")
AddReference("QuantConnect.Algorithm")
AddReference("QuantConnect.Algorithm.Framework")

from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework import *
from QuantConnect.Algorithm.Framework.Portfolio import PortfolioTarget
from QuantConnect.Algorithm.Framework.Risk import RiskManagementModel



class TakeProfitsPerPair(RiskManagementModel):
  
    def __init__(self, minimumReturnPercent = config.TakeProfit):
        self.TakeProfit = minimumReturnPercent 
        
    def ManageRisk(self, algorithm, targets):
        targets = []
        for target in targets:
            
            Pair = (target.Symbol , config.LETFInformation[target.Symbol].HedgingSymbol)
        
            pnl1 = algorthim.Securities[Pair[0]].Holdings.UnrealizedProfitPercent
            pnl2 = algorthim.Securities[Pair[1]].Holdings.UnrealizedProfitPercent
            if pnl1 + pnl2 > self.TakeProfit:
                # liquidate
                algorith.Debug("took profits")
                targets.append(PortfolioTarget(Pairs[0], 0))
                targets.append(PortfolioTarget(Pairs[1], 0))
            else:
                #keep old target
                targets.append(target)
                
        return targets
#region imports
from AlgorithmImports import *
#endregion
import numpy as np
import pandas as pd
from UniverseHelpers import LoadSymbolData


def Resample(prices, frequency):
    return prices[0::frequency]

def CummulativeReturn(discrete_returns):
    return (np.cumprod((1+discrete_returns)) -1).dropna()
                      
def DiscreteReturn(prices, timestep):
    if timestep == 1: return prices.pct_change().dropna()
    else:return Resample(prices,timestep).pct_change().dropna()
    

class SpreadData:
    def __init__(self, RelevantHistory ,BarSize, price_feature, information):
        
    
        self.BarSize = BarSize
        self.Information = information
 
        self.RawResampledCloseData = RelevantHistory.loc[:,price_feature].unstack(level = 0).dropna()[0::BarSize]
        self.RawResampledVolumeData = RelevantHistory.loc[:,"volume"].unstack(level = 0).dropna()[0::BarSize]
        self.SpreadData = pd.DataFrame()
        self.DiscreteReturns = pd.DataFrame()
        self.DailyIntradayReturns = pd.DataFrame()
        self.UniqueDays = pd.Series(self.RawResampledCloseData.index.date).unique()
        
        DiscreteReturns = []
        CummulativeReturns = []
        RVs = []
        for unique_day in self.UniqueDays:
            
            today =  self.RawResampledCloseData[self.RawResampledCloseData.index.date == unique_day]
            DiscreteReturns.append(today.pct_change().dropna())
            CummulativeReturns.append((1+today.pct_change().dropna()).cumprod()-1)
            rv = DiscreteReturns[-1].apply(lambda x: x**2).cumsum()
            RVs.append(rv)
        self.DiscreteReturns = pd.concat(DiscreteReturns)
        self.CummulativeReturns = pd.concat(CummulativeReturns)
        self.RV = pd.concat(RVs)
        
        
    def Spreads(self,Pair, daily= False):
        df = pd.DataFrame(
            columns = [
                "BullSpread", "BearSpread", "PairSpread","Benchmark", 
                "DailyMean_BearSpread" , "DailySwing_BearSpread",
                "DailyMean_BullSpread" , "DailySwing_BullSpread",
                 "DailyMean_PairSpread" , "DailySwing_PairSpread"])
                 
        bull_ticker, bear_ticker = Pair
        benchmark_ticker =  self.Information[bull_ticker].TrackingBenchmark
        df["Benchmark"] = self.CummulativeReturns[benchmark_ticker]
        df["Bull"] =  self.CummulativeReturns[bull_ticker]
        df["Bear"] =  self.CummulativeReturns[bear_ticker]
        
        df["BullSpread"] = self.CummulativeReturns[bull_ticker] -self.Information[bull_ticker].Beta * df["Benchmark"]
        
        df["ObservedBeta_Bull"] =   self.CummulativeReturns[bull_ticker]/df["Benchmark"]
        df["Observed_Beta_Bear"] =   self.CummulativeReturns[bear_ticker]/df["Benchmark"]
        df["BearSpread"] = self.CummulativeReturns[bear_ticker] -self.Information[bear_ticker].Beta * df["Benchmark"]
        df["PairSpread"] =  self.CummulativeReturns[bull_ticker] + self.CummulativeReturns[bear_ticker]
        df["RV"] = self.RV[benchmark_ticker]
        #df["VIX"] = Returns["VIXM"]
        
 
    
        df["Corr_Bear"] = df["Benchmark"].expanding().corr(df["BearSpread"])
        df["Corr_Pair"] = df["Benchmark"].expanding().corr(df["PairSpread"])
        df["Corr_Bull"] = df["Benchmark"].expanding().corr(df["BullSpread"])
        if daily:
            for unique_day in self.UniqueDays:
                df.loc[df.index.date ==unique_day,"DailyMean_BullSpread"] =  np.mean(df[df.index.date ==unique_day]["BullSpread"])
                df.loc[df.index.date ==unique_day,"DailyMean_BearSpread"] =  np.mean(df[df.index.date ==unique_day]["BearSpread"])
                df.loc[df.index.date ==unique_day,"DailyMean_PairSpread"] =  np.mean(df[df.index.date ==unique_day]["PairSpread"])
                
                df.loc[df.index.date ==unique_day,"DailySwing_BullSpread"] =  max(df[df.index.date ==unique_day]["BullSpread"]) - min(df[df.index.date ==unique_day]["BullSpread"])
                df.loc[df.index.date ==unique_day,"DailySwing_BearSpread"] =  max(df[df.index.date ==unique_day]["BearSpread"]) - min(df[df.index.date ==unique_day]["BearSpread"])
                df.loc[df.index.date ==unique_day,"DailySwing_PairSpread"] =  max(df[df.index.date ==unique_day]["PairSpread"]) - min(df[df.index.date ==unique_day]["PairSpread"])
        
        
        return df 
        
    def OverallBenchmark(self,Pair):
        benchmark_ticker =  self.Information[Pair[0]].TrackingBenchmark
        return (1+self.RawResampledData[benchmark_ticker].pct_change()).cumprod() 
        
   
       
#####################################################
#region imports
from AlgorithmImports import *
#endregion
''' LETFData holds all relevant fundamentals needed to build its signal '''
class LETFData:
    
    def __init__(self,symbol,benchmark,beta, opposite):
        
        self.TrackingBenchmark = benchmark
        self.Beta = beta
        self.HedgingSymbol = opposite
      

'''

LoadSymbolData() takes in list of SubUniverses ( dictionaries that are manually defined in Information.py and stored as objects ) 

The function returns two objects:

1) dict SymbolDataDictionary[LETFTicker:LETFData] -  maps an LETFTicker to its LETFData. This object will be globally exposed in Main.py and LETFAlphaModel in order to 
have to quick access to fundamentals information.


2) list PairsList[(BullETF_Beta1:BearETF_-Beta1)] - Lists of tuples holdings the tickers that would constitute a Pairs Trade 

These objects only have to be created once at runtime, and it simplifies the passing of information within the self.
'''


def LoadSymbolData(dict_list):

    SymbolData = {} #1
    PairsList = [] #2
    
    # iterate over each individual SubUniverse's informaton dictionary
    for info_dict in dict_list:    
        # Then there is more than One Pair and I manaully set the Pairs in a nested dictionary that is retreived with the key "Trade"
        if "Trade"  in info_dict.keys():
            #BullETF_Beta:BearETF_-Beta is the format of the what .items() returns
            for ticker1, ticker2 in info_dict["Trade"].items():
                # Append a tupple of the Pair tickers which we will need later in the AlphaModels.Update() method.
                PairsList.append((ticker1,ticker2))
                bench = info_dict["Benchmark"]
                
                    
                SymbolData[ticker1] = LETFData(
                    symbol = ticker1, 
                    benchmark = info_dict["Benchmark"], 
                    # The Beta of an LETF is found within the Bull/Bear ETF subdictionaries. "Trade" is conventional and manually written in Bull:Bear format. 
                    beta = info_dict["BullETFs"][ticker1], 
                    opposite = ticker2)
                    
                SymbolData[ticker2] = LETFData(
                    symbol = ticker2,
                    benchmark = info_dict["Benchmark"],
                    beta = info_dict["BearETFs"][ticker2],
                    opposite = ticker1
                    )
                    
            if info_dict["Benchmark"] not in SymbolData.keys():
                SymbolData[bench]= LETFData(
                        symbol = bench,
                        benchmark = info_dict["Benchmark"],
                        beta = 1,
                        opposite = None
                        ) 
        else: #only 1 pair
            bear = list(info_dict["BearETFs"].keys())[0]
            bull = list(info_dict["BullETFs"].keys())[0]
            bench =  info_dict["Benchmark"]
            
            PairsList.append((bull,bear))
            
            SymbolData[bench]= LETFData(
                    symbol = bench,
                    benchmark = info_dict["Benchmark"],
                    beta = 1,
                    opposite = None
                    )
                    
            SymbolData[bull]= LETFData(
                    symbol = bull,
                    benchmark = info_dict["Benchmark"],
                    beta = info_dict["BullETFs"][bull],
                    opposite = bear
                    )
                    
            SymbolData[bear] = (LETFData(
                    symbol = bear,
                    benchmark = info_dict["Benchmark"],
                    beta = info_dict["BearETFs"][bear],
                    opposite = bull))
                    
    return SymbolData, PairsList

    
    
def GetTickersFromUniverse(subuniverses_list, traded= True): 
    all_tickers = []
    for subuniverse in subuniverses_list:
        #automatically include the benchmark
        all_tickers.append(subuniverse["Benchmark"])
        # the defaultt setting where we are considering only Pairs we are interested in trading. Manually set in Information.py 
        if "Trade" in subuniverse.keys():
            for  key, val in subuniverse["Trade"].items():
                all_tickers.append(key)
                all_tickers.append(val)
         
        elif "Trade" not in subuniverse.keys() :
            all_tickers = all_tickers + (list(subuniverse['BearETFs'].keys()))
            all_tickers = all_tickers + (list(subuniverse['BullETFs'].keys()))
            
    return all_tickers
#region imports
from AlgorithmImports import *
#endregion
# Your New Python File
#region imports
from AlgorithmImports import *
#endregion
from clr import AddReference
AddReference("QuantConnect.Algorithm")
AddReference("QuantConnect.Indicators")
AddReference("QuantConnect.Algorithm.Framework")

from QuantConnect.Algorithm import QCAlgorithm
from QuantConnect.Data.UniverseSelection import *
from QuantConnect.Indicators import *
from QuantConnect import Market 

# Make changes in Configure
from Configure import LETFInformation, PairsList # LETF iterables for easy access.
from Configure import DiscountSpreadThreshold, PremiumSpreadThreshold, RollingWindowLength, BarSize, TakeProfit, FixDollarSize, TradingFrequency, WonkSpread, beta_
from Configure import asPair, LS
import numpy as np
import pandas as pd
from collections import deque

        
class LETFArb(QCAlgorithmFramework):

    def Initialize(self):
        self.SetStartDate(2016, 8, 1)  # Set Start Date
        self.SetEndDate(2022, 7, 1)
        self.SetBrokerageModel(BrokerageName.AlphaStreams)
        self.SetCash(170e6)
        self.b = 0
        #Holds the raw data. Updated with  UpdateQuoteBars() nested withing OnData
        self.ClosingPrices = {}
        self.Corrs = {}
        self.SpreadMeans = {}
        self.MostRecentSpread = {}
        
        for Pair in PairsList:
            self.Corrs[Pair] = []
        for symbol in LETFInformation.keys():
            equity = self.AddEquity(symbol, Resolution.Minute)
            self.ClosingPrices[symbol] = []
            self.SpreadMeans[symbol] = RollingWindow[float](RollingWindowLength*100)
            
            
        self.SetExecution(ImmediateExecutionModel())
        self.Settings.FreePortfolioValuePercentage = 0.025
        
        equity = self.AddEquity("SPY", Resolution.Minute)
        
        self.SetBenchmark("SPY")
        equity = self.AddEquity("VXX", Resolution.Minute)
        equity = self.AddEquity("VIXY", Resolution.Minute)
        symbols = []
        for symbol in LETFInformation.keys():
            symbols.append(Symbol.Create(symbol, SecurityType.Equity, Market.USA))
        self.SetUniverseSelection(ManualUniverseSelectionModel(symbols))
     
            
        ### Scheduled Events to handle logic and risk managment is intuitive to me ###
        
        # ManageBars - reset daily rolling window, update volatility lookback window.
        
        self.Schedule.On(
            self.DateRules.EveryDay("SPY"), 
            self.TimeRules.Every(timedelta(minutes=TradingFrequency)), self.Trade)
        
        self.Schedule.On(
            self.DateRules.EveryDay("SPY"), 
            self.TimeRules.BeforeMarketClose("SPY",2), self.ResetTradeBars)
            
        self.Schedule.On(
            self.DateRules.EveryDay("SPY"), 
            self.TimeRules.Every(timedelta(minutes=1)), self.CheckProfits)
        """ 
        self.Schedule.On(
            self.DateRules.EveryDay("SPY"), 
            self.TimeRules.BeforeMarketClose("SPY",90), self.Trade)
        """    
          
    
    def OnData(self, data):  
        
        
        tradeBars = data.Bars
        for Ticker in self.ClosingPrices.keys():
            if not tradeBars.ContainsKey(Ticker): 
                return
        for Ticker in self.ClosingPrices.keys():
            prices = self.ClosingPrices[Ticker]
            self.Debug(type(prices))
            price = tradeBars[Ticker].Close
            prices.append(price)
            self.ClosingPrices[Ticker] = prices
            
    
    def CheckProfits(self):
        for Pair in PairsList:
                
                tickers= [Pair[0], Pair[1], 'VXX', "VIXY"]
                BearTicker, BullTicker = Pair[0], Pair[1]
                    #if self.Portfolio[BenchmarkTicker].UnrealizedProfitPercent < -.1: 
                      #self.MarketOrder("URTY",100)
                pair_earn = (self.Portfolio[Pair[0]].UnrealizedProfit + self.Portfolio[Pair[1]].UnrealizedProfit)/ self.Portfolio.TotalPortfolioValue
                
                def _check(ticker):
                    holding = self.Portfolio[ticker]
                    value = holding.AbsoluteHoldingsValue
                    earn = holding.UnrealizedProfitPercent
                    #opp = [tic for tic in tickers if tic != ticker ][0]
                    
                    if earn > TakeProfit:
                        self.Debug(f"{ticker} earned {earn} on {self.Time}- Closed")
                        
                        #self.SetHoldings(ticker,-.25)
                        self.SetHoldings(ticker,0)
                    elif -.1 <earn <-1*TakeProfit:
                        
                        if holding.IsLong and self.MostRecentSpread[ticker] >PremiumSpreadThreshold:
                            
                            
                            self.LimitOrder_FixedDollar(ticker, value)
                            self.Debug(f"{ticker} down {earn} at {self.Time} while long - Barrelled Small")
                            
                        elif holding.IsShort and self.MostRecentSpread[ticker] <DiscountSpreadThreshold:
                            
                            self.LimitOrder_FixedDollar(ticker, -1* value)
                            self.Debug(f"{ticker} down {earn} at {self.Time} while Short - Barrelled Small")
                        
                    elif earn <-.10:
                        if holding.IsLong and self.MostRecentSpread[ticker] >PremiumSpreadThreshold:
                            self.LimitOrder_FixedDollar(ticker, 2*value)
                            self.Debug(f"{ticker} down {earn} at {self.Time} while long - Barrelled Small")
                            
                        elif holding.IsShort and self.MostRecentSpread[ticker] <DiscountSpreadThreshold:
                            self.LimitOrder_FixedDollar(ticker, -2* value)
                            self.Debug(f"{ticker} down {earn} at {self.Time} while Short - Barrelled Small")   
                        else:
                            self.Debug(f"{ticker} earned {earn} on {self.Time} - Standby")
                        
                
                if not asPair:        
                    for ticker in tickers: 
                        _check(ticker)
                else:
                    if pair_earn > TakeProfit:
                        self.Debug(f"{Pair[0]} earned {self.Portfolio[Pair[0]].UnrealizedProfit/ self.Portfolio.TotalPortfolioValue} on {self.Time}")
                        self.Debug(f"{Pair[1]} earned {self.Portfolio[Pair[1]].UnrealizedProfit/ self.Portfolio.TotalPortfolioValue} on {self.Time}")
                        self.SetHoldings(Pair[0],0)
                        self.SetHoldings(Pair[1],0)
                
               
                  
                    
        #    
        #if (bench_earn) > TakeProfit :
         #   self.Liquidate(BenchmarkTicker)
          #  self.Debug(f"Benchmark Earned {bench_earn}  Time: {self.Time}")
        #                
        
            #_check(Pair[0])
           
    def Trade(self):
        
        for Pair in PairsList:
            
            BullTicker, BearTicker, BenchmarkTicker = Pair[0], Pair[1], LETFInformation[Pair[0]].TrackingBenchmark
            def Resample(prices, frequency):
                return prices[0::frequency]
                    
            BullPrices = Resample(self.ClosingPrices[BullTicker],BarSize)
            BearPrices = Resample(self.ClosingPrices[BearTicker],BarSize)
            BenchPrices = Resample(self.ClosingPrices[BenchmarkTicker],BarSize)
            
            if len(BullPrices) < 5 or len(BearPrices) < 5 or len(BenchPrices) <5:
                return
        
                
            bullspreads = self.GetSpread(BullPrices, BenchPrices,LETFInformation[BullTicker].Beta)
               
            bearspreads = self.GetSpread(BearPrices, BenchPrices,LETFInformation[BearTicker].Beta)
            
                 
            #Signal =  (bearspreads + -1 * bullspreads)
            """
            Corr = np.corrcoef(Signal, self.CummulativeReturn(BullPrices)+self.CummulativeReturn(BearPrices))[0][1]
            old_corrs = self.Corrs[Pair]
            old_corrs.append(Corr)
            self.Corrs[Pair] = old_corrs
                
            CorZ = (Corr- np.mean(old_corrs))/ np.std(old_corrs)
            """ 
            
            """ 
            Spreads arise from excess momentum. 
                LETFs are short term speculative instruments, and should carry information about momementum.
                
                What does a relative premium in the 3x Bull ETF say? Market may be overbought right now.
                What does a relative discount in the 3x Bear ETF say? Same thing.
                
                Spreads are legally managed by Authorized Participants.
                In laymans terms, Spreads arise from price action that market makers don't correct for.
                
                We measured that relative Spreads of opposite Beta LETFs should are also stationary and mean reverting to 0. 
                
                When markets are functioning well, Spreads are low. When they are misbehaving, Spreads are high because 
                speculation is rampant and market makers step out. When specualtors take over the market, Spreads should
                correlate to benchmark price action.
            """
            
            #NoCorr =  abs(CorZ) <=0.01
            #HighCorr =  (CorZ) >=2
            #PositiveCorr =  Corr >= .4
            #NegativeCorr = Corr<= -.4 
            #NormalCorr = (NoCorr== False) & ( PositiveCorr ==False) & (NegativeCorr== False)
       
                
            bench_invested=self.Securities[BenchmarkTicker].Invested
            
            bull_invested= self.Securities[BullTicker].Invested
            bear_invested= self.Securities[BearTicker].Invested
                
            self.MostRecentSpread[BullTicker] = bullspreads.iloc[-1]
            self.MostRecentSpread[BearTicker] = bearspreads.iloc[-1] 
            
            self.SpreadMeans[BullTicker].Add(self.MostRecentSpread[BullTicker])
            self.SpreadMeans[BearTicker].Add(self.MostRecentSpread[BearTicker])
            
            #bear_ts = pd.Series(list(self.SpreadMeans[BearTicker]))
            #bull_ts = pd.Series(list(self.SpreadMeans[BullTicker]))
            
            
            """
            BullZ = bullspread
            BearZ = bearspread
            """
            """
            if Corr > .9: 
                
                self.Debug(f"{Corr} betweeen Benchmark and Excess Bearish Momentum at {self.Time}")
                #self.Liquidate(BearTicker)
                    
                #self.SetHoldings(BearTicker,.5)
                if not self.Portfolio[BullTicker].Invested:  
                    if not self.Portfolio[BearTicker].Invested:
                        self.MarketOrder(BullTicker,  self._dollar_to_shares(BullTicker,FixDollarSize))
                    else: 
                        #self.delta_neutral_entry(new = BullTicker, existing_leg = BearTicker)
                        self.Liquidate(BearTicker)
                else: 
                    if self.Portfolio[BullTicker].UnrealizedProfitPercent < -1*TakeProfit: self.delta_neutral_entry(new = BullTicker, existing_leg = BullTicker)
                 
                #self.SetHoldings(BenchmarkTicker,1)
            
            
            elif Corr < -.9:
                self.Debug(f"{Corr} betweeen Benchmark and Excess Bearish Momentum at {self.Time}")
                self.Liquidate(BenchmarkTicker)
                
                if not self.Portfolio[BearTicker].Invested:
                    if not self.Portfolio[BullTicker].Invested:
                        self.MarketOrder(BearTicker,  self._dollar_to_shares(BearTicker,FixDollarSize))
                    else: 
                        #self.delta_neutral_entry(new = BearTicker, existing_leg = BullTicker)
                        self.Liquidate(BullTicker)
                
                else:  
                    if self.Portfolio[BearTicker].UnrealizedProfitPercent < -1*TakeProfit:self.delta_neutral_entry(new = BearTicker, existing_leg = BearTicker)
                
                #self.SetHoldings(BearTicker,1,True) 
                
                
            elif not self.Portfolio.Invested: 
                self.SetHoldings(BenchmarkTicker,1)
            
            """
            
            vix_invested = self.Securities["VIXY"].Invested
            BettingSize =  FixDollarSize * self.Portfolio.TotalPortfolioValue
            """
            # Short-vol, trend-following off
            if min(BearZ, BullZ) <DiscountSpreadThreshold and max(BearZ, BullZ) < DiscountSpreadThreshold/2:
                    #self.SetHoldings(BearTicker,-.20)
                #self.SetHoldings("SPY",.5)
                #self.SetHoldings(BullTicker,.75)
                #self.SetHoldings("VXX", -.5)
                self.Debug(f"Discounted Vol at {self.Time} ")
                self._reportspread(BullTicker,bullspread)
                self._reportspread(BearTicker,bearspread)
               
               
                #self.SetHoldings("VIXY",.3)
                self.SetHoldings(BullTicker, -.5)
                self.SetHoldings(BearTicker,-.5)
                
                #self.SetHoldings("VIXY",-.25)
                #self.SetHoldings(BullTicker,1,True)
                #self.Liquidate("SPY")
                #self.MarketOrder(BearTicker,1000)
                #self.MarketOrder(BullTicker,1000)
                #self.SetHoldings(BullTicker,.5)
                #self.SetHoldings(BearTicker,.5) 
                #self.Liquidate(BenchmarkTicker)
                #self.MarketOrder(BullTicker,  self._dollar_to_shares(BullTicker,BettingSize))
                #self.MarketOrder(BearTicker,  self._dollar_to_shares(BearTicker,BettingSize))
                if beta_[0]:
                    self.SetHoldings(BenchmarkTicker, beta_[1])
                
                #self.SetHoldings(BearTicker,-.5)
                #self.SetHoldings(BullTicker,-.5)
                #self.SetHoldings(BearTicker,.25)
                #self.SetHoldings(BearTicker,.25) 
                #self.Liquidate(BenchmarkTicker)https://www.quantconnect.com/project/5360744#optimization-view-tab
                #self.SetHoldings(BenchmarkTicker, 1)
                #if bear_invested:
                #    self.Liquidate(BearTicker)
                    
                #self.SetHoldings(BearTicker,.5)
            #Momentum
            
            
            if min(BearZ, BullZ) <DiscountSpreadThreshold and max(BearZ, BullZ) > PremiumSpreadThreshold: 
                    #self.SetHoldings(BearTicker,-.20)
                #self.SetHoldings("SPY",.5)
                #self.SetHoldings(BullTicker,.75)
                #self.SetHoldings("VXX", -.5)
                self.SetHoldings(BullTicker, -.5)
                self.SetHoldings(BearTicker,-.5)
                #self.SetHoldings("VXX",.5)
                #self.SetHoldings(BullTicker,1,True)
                #self.Liquidate("SPY")
                #self.MarketOrder(BearTicker,1000)
                #self.MarketOrder(BullTicker,1000)
                #self.SetHoldings(BullTicker,.5)
                #self.SetHoldings(BearTicker,.5) 
                #self.Liquidate(BenchmarkTicker)
                #self.MarketOrder(BullTicker,  self._dollar_to_shares(BullTicker,BettingSize))
                #self.MarketOrder(BearTicker,  self._dollar_to_shares(BearTicker,BettingSize))
                if beta_[0]:
                    self.Debug("Discount with Momentum skew to Bullish")
                    self.SetHoldings(BenchmarkTicker, beta_[1])
                if BullZ> PremiumSpreadThreshold:
                    self.Debug(f"Discount with Momentum skew to Bullish at {self.Time}")
                    #self.SetHoldings(BullTicker,.25)
                    self.SetHoldings(BearTicker,.25)
                    
                elif BearZ >PremiumSpreadThreshold:
                    self.Debug(f"Discount with Momentum skew to Bearish at {self.Time}")
                    self.SetHoldings(BearTicker,.25)
                    self.SetHoldings(BullTicker,.25)
                    
                    
                self._reportspread(BullTicker,bullspread)
                self._reportspread(BearTicker,bearspread)
                #self.SetHoldings(BearTicker,.25) 
                #self.Liquidate(BenchmarkTicker)
                #self.SetHoldings(BenchmarkTicker, 1)
                #if bear_invested:
                #    self.Liquidate(BearTicker)
                    
                #self.SetHoldings(BearTicker,.5)
            """    
            if (self.MostRecentSpread[BearTicker] < DiscountSpreadThreshold) and (self.MostRecentSpread[BullTicker] >PremiumSpreadThreshold) :
                self._reportspread(BullTicker,self.MostRecentSpread[BullTicker])
                self._reportspread(BearTicker,self.MostRecentSpread[BearTicker])
                if self.Portfolio[BearTicker].IsLong:
                        ticket = self.LimitOrder_FixedDollar(BearTicker, -2*BettingSize)
                if self.Portfolio[BullTicker].IsShort:
                        ticket = self.LimitOrder_FixedDollar(BearTicker, 2*BettingSize)
                        #self.Debug(f'Closing out a Long for SQQQ at {self.Time}')
                else:       
                    if not(bull_invested or bear_invested):
                        ticket = self.LimitOrder_FixedDollar(BearTicker, -1*BettingSize)
                        self.LimitOrder_FixedDollar(BullTicker, BettingSize)
                        #self.Debug(f'New Short Entry for SQQQ at {self.Time}')
                    else:
                        if bear_invested: self.LimitOrder_FixedDollar(BearTicker, -1*BettingSize)
                        if bull_invested: self.LimitOrder_FixedDollar(BullTicker, BettingSize)
                        #self.Debug(f'Adding to Short for SQQQ at {self.Time}')     
                    
            if (self.MostRecentSpread[BearTicker] > PremiumSpreadThreshold) and (self.MostRecentSpread[BullTicker] < DiscountSpreadThreshold):
                self._reportspread(BullTicker,self.MostRecentSpread[BullTicker])
                self._reportspread(BearTicker,self.MostRecentSpread[BearTicker])
                if self.Portfolio[BearTicker].IsShort:
                        ticket = self.LimitOrder_FixedDollar(BearTicker, -2*BettingSize)
                if self.Portfolio[BullTicker].IsLong:
                        ticket = self.LimitOrder_FixedDollar(BullTicker, -2*BettingSize)
                        #self.Debug(f'Closing out a Short for SQQQ at {self.Time}')
                else:       
                    if not bear_invested:
                        ticket = self.LimitOrder_FixedDollar(BearTicker, -1*BettingSize)
                        #self.Debug(f'New Long Entry for SQQQ at {self.Time}')
                    else:
                        ticket = self.LimitOrder_FixedDollar(BearTicker, -1*BettingSize)
                        #self.Debug(f'Adding to Long for SQQQ at {self.Time}')    

            """
            """
            """
            """
    def InvestedAndProfited(self,Ticker):
        if self.Portfolio[Ticker].UnrealizedProfitPercent > 0.07:
            self.Debug("Took Profits {} at {}-{}".format(Ticker,self.Portfolio[Ticker].UnrealizedProfitPercent, self.Time))
            self.MarketOrder(Ticker, -1* self.Portfolio[Ticker].Quantity)
            
        elif self.Portfolio[Ticker].UnrealizedProfitPercent<-.99:
            self.Debug("In the hole {}- {}".format(Ticker, self.Time))
            self.MarketOrder(Ticker, .00001*self.Portfolio[Ticker].Quantity)

        
    def CummulativeReturn(self,ts):
        return (1+pd.Series(ts).pct_change().dropna()).cumprod()-1
    
    
    def GetSpread(self,letf_ts,benchmark_ts, Beta):

        cummuative_letf_ts =  self.CummulativeReturn(letf_ts)
        cummulative_bench_ts = self.CummulativeReturn(benchmark_ts)
        expected_letf_ts =  cummulative_bench_ts * Beta
        spread = cummuative_letf_ts - expected_letf_ts  
        
        
        return spread
    
    def ResetTradeBars(self):
        for Ticker in self.ClosingPrices.keys():
            self.ClosingPrices[Ticker] = []
            if self.Portfolio[Ticker].Invested:
                #self.SetHoldings(Ticker,0)
                #self.Debug(f"Liqudated {Ticker}")
                pass
                
                
    def _reportspread(self, ticker,spread):
        self.Debug(f"{ticker} has {spread} Spread as {self.Time}")
  
    
    def _dollar_to_shares(self,ticker,dollar_size):
        return round(dollar_size / self.Securities[ticker].Price)
    
    def LimitOrder_FixedDollar(self,ticker,dollars):
        last_price = self.Securities[ticker].Price
        if dollars <0:
            ticket = self.LimitOrder(ticker, 
            self._dollar_to_shares(ticker, dollars), 
            last_price +.01)

        else:
            ticket = self.LimitOrder(ticker, 
            self._dollar_to_shares(ticker, dollars), 
            last_price -.01)
        
        return ticket

    def dilute_position(self,ticker):
        
        if self.Portfolio[ticker].UnrealizedProfitPercent < -1* TakeProfit:
            self.MarketOrder(ticker,  self._dollar_to_shares(ticker, FixDollarSize ))
            
    
    def _dollar_to_weight(self, dollars):
        
        pv= dollars/self.Portfolio.TotalPortfolioValue
    
    def delta_neutral_entry(self, new, existing_leg):
        new_dollars = self.Portfolio[existing_leg].Quantity * self.Portfolio[existing_leg].Price 
        self.MarketOrder(new,  self._dollar_to_shares(new,new_dollars))
        
        if new != existing_leg: self.Debug(f"Delta Hedged {existing_leg} with {new}")
#region imports
from AlgorithmImports import *
#endregion
from clr import AddReference
AddReference("QuantConnect.Algorithm")
AddReference("QuantConnect.Indicators")
AddReference("QuantConnect.Algorithm.Framework")

from QuantConnect.Algorithm import QCAlgorithm
from QuantConnect.Data.UniverseSelection import *
from QuantConnect.Indicators import *
from QuantConnect import Market 

# Make changes in Configure
from Configure import LETFInformation, PairsList, DirectionalTrading, VolTrading # LETF iterables for easy access.
from Configure import DiscountSpreadThreshold, PremiumSpreadThreshold, RollingWindowLength, BarSize, TakeProfit, FixDollarSize, TradingFrequency, WonkSpread, beta_
from Configure import asPair, LS, AntiSpread
from Configure import RebarrelThreshold, BetterExecution
import numpy as np
import pandas as pd
from collections import deque

        
class LETFArb(QCAlgorithmFramework):

    def Initialize(self):
        self.SetStartDate(2015, 1, 1)  # Set Start Date
        self.SetEndDate(2022, 9, 24)
        self.SetBrokerageModel(BrokerageName.AlphaStreams)
        self.SetCash(170e6)
        self.b = 0
        #Holds the raw data. Updated with  UpdateQuoteBars() nested withing OnData
        self.ClosingPrices = {}
        self.Corrs = {}
        self.SpreadMeans = {}
        self.MostRecentSpread = {}
        self.reported  = {}
        for Pair in PairsList:
            self.Corrs[Pair] = []
        for symbol in LETFInformation.keys():
            equity = self.AddEquity(symbol, Resolution.Minute)
            self.ClosingPrices[symbol] = []
            self.SpreadMeans[symbol] = RollingWindow[float](RollingWindowLength*100)
            
            
        self.SetExecution(ImmediateExecutionModel())
        self.Settings.FreePortfolioValuePercentage = 0.025
        
        equity = self.AddEquity("SPY", Resolution.Minute)
        
        self.SetBenchmark("SPY")
        equity = self.AddEquity("VXX", Resolution.Minute)
        equity = self.AddEquity("VIXY", Resolution.Minute)
        equity = self.AddEquity("TVIX", Resolution.Minute)
        symbols = []
        for symbol in LETFInformation.keys():
            symbols.append(Symbol.Create(symbol, SecurityType.Equity, Market.USA))
        self.SetUniverseSelection(ManualUniverseSelectionModel(symbols))
     
            
        ### Scheduled Events to handle logic and risk managment is intuitive to me ###
        
        # ManageBars - reset daily rolling window, update volatility lookback window.
        """
        self.Schedule.On(
            self.DateRules.EveryDay("SPY"), 
            self.TimeRules.At(9,45), self.DoubleShort)
        """
        self.Schedule.On(
            self.DateRules.EveryDay("SPY"), 
            self.TimeRules.At(15,55), self.DoubleShort)
  
    def DoubleShort(self):
       # self.SetHoldings('SQQQ',-.49)
       # self.SetHoldings('TQQQ',-.49)  
        #self.SetHoldings('RUSL',.10)    
        #self.SetHoldings('RUSS',.10)  
        self.SetHoldings('YINN',-.40)    
        self.SetHoldings('YANG',-.40)
        #self.SetHoldings('QQQ',.03)
        #self.SetHoldings('VIXY',-.1)


    def OnData(self, data):  
        pass
     
#region imports
from AlgorithmImports import *
#endregion
from clr import AddReference
AddReference("QuantConnect.Algorithm")
AddReference("QuantConnect.Indicators")
AddReference("QuantConnect.Algorithm.Framework")

from QuantConnect.Algorithm import QCAlgorithm
from QuantConnect.Data.UniverseSelection import *
from QuantConnect.Indicators import *

from Execution.ImmediateExecutionModel import ImmediateExecutionModel
from collections import deque, UserDict
import numpy as np


ll = 0
ul = 3

class Universe(UserDict):
    def __delitem__(self, key):
        pass
    def __setitem__(self, key, value):
        pass


        

EmergingMarkets = [
    ("EFO","EFU",-1,3,3), #Proshares MSCI EAFE
    ("UPV","EPV",-1,3,3), #Proshares MSCI Developed EU
    ("FXP","XPP",-1,3,3), #Proshares MSCI China
    ("EWV","EZJ",-1,3,3)] #Proshares MSCI Japan]
    

ProsharesSectorETF =  [  
    ("UYM","SMN",-1,3,3), #Proshares Dow Jones U.S. Basic Materials    
    ("UBIO","ZBIO",-1,3,3), #Proshares Nasdaq Biotech  3x
    ("BIB","BIS",-1,3,3), #Proshares Nasdaq Biotech  2x
    ("SCOM","UCOM",-1,3,3), #Proshares S&P Communication Services Select Sector  3x
    ("SKF","UYG",-1,3,3), #Proshares Dow Jones U.S. Financials
    ("FINU","FINZ",-1,3,3), #Proshares S&P Financial Select Sector
    ("RXD","RXL",-1,3,3), #Proshares Dow Jones U.S. Health Care
    ("UXI","SIJ",-1,3,3), #Proshares Dow Jones U.S. Industrials
    ("DIG","DUG",-1,3,3), #Proshares Dow Jones U.S. Oil & Gas
    ("SRS","URE",-1,3,3), #Proshares Dow Jones Real Estate
    ("USD","SSG",-1,3,3), #Proshares Dow Jones U.S. Semiconductors
    ("ROM","REW",-1,3,3), #Proshares Dow Jones U.S. Technology
    ("SDP","UPW",-1,3,3)]     

NotLiquid = [
    ("SAA", "SDD"),
    ("MZZ", "MVV", -1,3,3),
    ("UMDD", "SMDD", -1,3,3),
     ("GLL","UGL",-1,3,3),#Proshares Bloomberg Gold Subindex
      ("AGQ","ZSL",-1,3,3),#Proshares Bloomberg Silver Subindex 
         ("YCS","YCL",-1,3,3),
         ("DSLV","USLV",-1,3,3), 
    ("UGLD","DGLD",-1,3,3),
    ("GUSH","DRIP",-1,3,3), #Direxion Oils and Gas Exploration
    ("RUSL","RUSS",-1,3,3), #Direxion Russia
    ("GASL","GASX",-1,3,3), #Direxion Natural Gas
    ("FAZ","FAS",-1,3,3),#Direxion Financials
     ("ERY","ERX",-1,3,3), #Direxion Energy
      ("YINN","YANG",-1,3,3)
      
    ] + EmergingMarkets + ProsharesSectorETF

USTreasury = [
  ("TBT","UBT",-1,3,3), #Proshares ICE U.S. Treasury 20+ Year Bond
  ("PST","UST",-1,3,3), #Proshares ICE U.S. Treasury 7 Year Bond
  ("TMF","TMV",-1,3,3)]
  
LiquidETFCompetition = [
    ("UGAZ","DGAZ",-1,3,3),
("ERY","ERX",-1,3,3),  
("NUGT","DUST",-1,3,3),
("UCO","SCO",-1,3,3),
("NUGT","DUST",-1,3,3),
("TECS","TECL",-1,3,3),
("SOXS","SOXL",-1,3,3)]

SP500 = [  #Proshares SP Small Cap
     #Proshares SP Mid Cap 2x 
     #Proshares SP Mid Cap 3x
    ("SPY", "SH", -1, 3,3), #-1
    ("SDS","SSO",-1,3,3),#Proshares SP500 2x
    ("UPRO","SPXU",-1,3,3), #3x
    ("SPXL","SPXS",-1,3,3)]# 3x

NASDAQ = [ 
    ("TQQQ","SQQQ",-1,2,2), #Proshares Nasdaq 3x
    ("QQQ","PSQ",-1,2,2 ), #1x
    ("QLD","QID",-1,2,2)] #2x
    

Russell2000 = [
    ("SRTY","URTY",-1,ul,ll), #Proshares Russel 3x
    ("RWM","IWM",-1,ul,ll), #Proshares Russel 1x
    ("TWM","UWM",-1,ul,ll)]
    
DirexionETFs = [ 
  ("TECL","TECS",-1,ll,ul),#Direxion Tech 3x
  ("TNA","TZA",-1,ll,ul), #Direxion Small Cap 3x
  ("LABU","LABD",-1,ll,ul), #Direxion Biotech
  ("NUGT","DUST",-1,ll,ul), #Direxion Gold Miners
  ("JNUG","JDST",-1,ll,ul) #Direxion Junior Gold Miners
 ] 
  
Commoditities = [
     ("OILU","OILD",-1,ll,ul), #Proshares Bloomberg WTI Crude Oil Subindex 3x
    ("UCO","SCO",-1,ll,ul),#Proshares Bloomberg WTI Crude Oil Subindex 2x
    ("ERY","ERX",-1,ll,ul)]  
    

def fetch_symbols(Pairs):
    symbols = []
    for info in Pairs:
        symbols.append(info[0])
        symbols.append(info[1])
    return symbols




DJIA  =  Universe()
DJIA.Benchmark = "DIA"
DJIA.Pairs =  [("DIA", 'DOG', -1, ll,ul), #Proshares Dow 1x
    ("SDOW","UDOW",-1),#Proshares Dow 3x
    ("DDM","DXD",-1)
    ] 
    
    
Russel  =  Universe()
Russel.Benchmark = "IWM"
Russel.Pairs =  [
    #("SRTY","URTY",-1,ul,ll), #Proshares Russel 3x
    ("RWM","IWM",-1,ul,ll), #Proshares Russel 1
    #("TWM","UWM",-1,ul,ll)
    ]

     


TradedUniverse = Russel

Bars = 15
PosSize =5000


RiskCap= -.5
Profit = .0003
MinSpread = 0
Z = .68
SlowVol = 30 #Days
BarLookBack = SlowVol*(6.5)*(60)/Bars
PairLookBack =  5


 

class LETFArb(QCAlgorithmFramework):

    def Initialize(self):
        self.SetStartDate(2015, 4, 1)  # Set Start Date
        self.SetEndDate(2019, 3, 2)
        BarPeriod = TimeSpan.FromMinutes(Bars)
       
        
        self.SetBrokerageModel(BrokerageName.AlphaStreams)
        self.BettingSize =  float(1/len(fetch_symbols(TradedUniverse.Pairs)))
        self.Debug(str(self.BettingSize))
        self.SetCash(round(PosSize/self.BettingSize))
    
        
        self.PriceData = {}
        equity = self.AddEquity("VXX", Resolution.Daily)
        self.VIX = RateOfChangePercent("VXX",Resolution.Daily)
        symbol = TradedUniverse.Benchmark
        equity = self.AddEquity(symbol, Resolution.Daily)
      
        
        for symbol in fetch_symbols(TradedUniverse.Pairs):
            equity = self.AddEquity(symbol, Resolution.Minute)
            self.PriceData[symbol] = deque(maxlen=2)
            
           
            
        self.Data = {}
    
       
      
        self.LETFSymbols = []
        for PairsInfo in TradedUniverse.Pairs:
            IndexConsolidator = TradeBarConsolidator(BarPeriod)
            LETFConsolidator= TradeBarConsolidator(BarPeriod)
            self.LETFSymbols.append(PairsInfo[1])
           
            
            data = Universe()
            data.LETFTicker =  PairsInfo[1]
            data.IndexTicker = PairsInfo[0]
            data.Leverage = PairsInfo[2]
            data.Spreads= deque(maxlen= int(BarLookBack))
            data.Pair = deque([],maxlen=PairLookBack)
            
            self.Data[data.LETFTicker] =  data
            
            
           
            IndexConsolidator.DataConsolidated += self.IndexHandler
            LETFConsolidator.DataConsolidated += self.LETFHandler
            
            self.SubscriptionManager.AddConsolidator(self.Data[data.LETFTicker].LETFTicker,LETFConsolidator)
            self.SubscriptionManager.AddConsolidator(self.Data[data.LETFTicker].IndexTicker,IndexConsolidator)
           
           
            
        self.SetExecution(ImmediateExecutionModel())
        self.SetBenchmark("SPY")
        self.IndexUpdated = False
        self.LETFUpdated = False
        symbols = []
        
        for symbol in fetch_symbols(TradedUniverse.Pairs):
            symbols.append(Symbol.Create(symbol, SecurityType.Equity, Market.USA))
        symbols.append(Symbol.Create("TVIX", SecurityType.Equity, Market.USA))
        self.SetUniverseSelection(ManualUniverseSelectionModel(symbols))
        
    
    
    def IndexHandler(self,sender, bar):
        
       
        try:
           Prices =  self.PriceData[bar.Symbol.Value]
        
           Prices.append(bar.Close)
           self.PriceData[bar.Symbol.Value] =  Prices
           self.IndexUpdated = True
            
        except KeyError:
            pass
        
    def LETFHandler(self,sender, bar):
        
        try:
           Prices =  self.PriceData[bar.Symbol.Value]
        
           Prices.append(bar.Close)
           self.PriceData[bar.Symbol.Value] =  Prices
           self.LETFUpdated = True
        except KeyError:
            pass
        
    def NowStale(self):
        self.IndexUpdated = False
        self.LETFUpdated = False
        
        
    def RecordPair(self,Data):
        Pair = Data.Pair
        IndexMV = self.Portfolio[Data.IndexTicker].Quantity * self.Portfolio[Data.IndexTicker].Price
        LETFMV = self.Portfolio[Data.LETFTicker].Quantity * self.Portfolio[Data.LETFTicker].Price
        Pair.append(IndexMV +LETFMV)
        Data.Pair = Pair
        
    def OnData(self, data):
        
        Updated = self.IndexUpdated and self.LETFUpdated
        if Updated:
            
            for key in self.LETFSymbols:
               
               Data = self.Data[key]
               LETFTicker =  Data.LETFTicker
               IndexTicker = Data.IndexTicker
               LETFPrices = self.PriceData[LETFTicker]
               IndexPrices = self.PriceData[IndexTicker]
               
               if len(LETFPrices) != 2: continue
               if len(IndexPrices) != 2: continue
               
               if LETFPrices[-2] !=0 and IndexPrices[-2] !=0:
                   LETFReturn = (LETFPrices[-1] - LETFPrices[-2])/ LETFPrices[-2]
                   IndexReturn = (IndexPrices[-1] - IndexPrices[-2])/ IndexPrices[-2]    
               
                   Spread = np.log(1+LETFReturn) - np.log(1+ Data.Leverage* IndexReturn)
                   Spreads = Data.Spreads
                   
                   
                   Spreads.append(Spread)
                   Data.Spreads = Spreads 
                   
               else: continue  
               
              
               
               OpenPosition = (self.Securities[Data.LETFTicker].Invested) and (self.Securities[Data.IndexTicker].Invested)
               if OpenPosition: self.RecordPair(Data)
                   
               
               
               if len(Data.Spreads) >= BarLookBack:
                    Spread = Data.Spreads[-1]
                    SpreadStds =  np.std(Data.Spreads)
                    Lowerband = -1*Z * SpreadStds
                    Upperband = Z* SpreadStds
                    Discount = Spread <= MinSpread and Spread < Lowerband
                    Premium = Spread >= abs(MinSpread) and Spread > Upperband
                    
               
                    if (Discount and not OpenPosition):
                           
                            
                            LETFInsight =  Insight.Price(LETFTicker, timedelta(Bars), InsightDirection.Up)
                            LETFInsight.Weight =  self.BettingSize
                            
                            IndexInsight =  Insight.Price(IndexTicker, timedelta(Bars), InsightDirection.Down)
                            IndexInsight.Weight =  self.BettingSize
                            
                            insights = [LETFInsight, IndexInsight]
                            
                            self.EmitInsights(Insight.Group(insights))
                            
                            self.SetHoldings([PortfolioTarget(LETFTicker, self.BettingSize), PortfolioTarget(IndexTicker, self.BettingSize)])
                            
                    if (Premium and OpenPosition):
                        self.EmitInsights(Insight.Price(Data.LETFTicker, timedelta(10), InsightDirection.Flat))
                        self.EmitInsights(Insight.Price(Data.IndexTicker, timedelta(10), InsightDirection.Flat))
                        self.Liquidate(Data.LETFTicker)
                        self.Liquidate(Data.IndexTicker)
                        Data.Pair = deque([], maxlen=int(PairLookBack))
            self.NowStale()
        else:
            for key in self.LETFSymbols:
                Data = self.Data[key]
            
                OpenPosition = (self.Securities[Data.LETFTicker].Invested) and (self.Securities[Data.IndexTicker].Invested)
                if OpenPosition:
                    self.RecordPair(Data)
                    Pair = Data.Pair
                    if len(Pair) == 0: continue
                    TotalReturn = (Pair[-1] - Pair[0])/Pair[0]
                    UnrealizedProfit = (self.Portfolio[Data.LETFTicker].UnrealizedProfitPercent + self.Portfolio[Data.IndexTicker].UnrealizedProfitPercent)/100
                    
                    if (UnrealizedProfit > Profit) or UnrealizedProfit< -.02: 
                        self.Debug("Early Exit: {}".format(UnrealizedProfit))                            
                        self.EmitInsights(Insight.Price(Data.LETFTicker, timedelta(10), InsightDirection.Flat))
                        self.EmitInsights(Insight.Price(Data.IndexTicker, timedelta(10), InsightDirection.Flat))
                        self.Liquidate(Data.LETFTicker)
                        self.Liquidate(Data.IndexTicker)
                        Data.Pair = deque([], maxlen=int(PairLookBack))
                    else:continue