From 2a413e29da4d1baf5382b898542bc7f8c67fbb65 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 8 Feb 2024 00:10:45 +0200 Subject: [PATCH 01/34] feat: remove template files --- DataProcessing/CLRImports.py | 12 - DataProcessing/DataProcessing.csproj | 30 --- DataProcessing/MyCustomDataDownloader.cs | 236 -------------------- DataProcessing/Program.cs | 81 ------- DataProcessing/config.json | 5 - DataProcessing/process.sample.ipynb | 78 ------- DataProcessing/process.sample.py | 32 --- DataProcessing/process.sample.sh | 0 Demonstration.cs | 77 ------- Demonstration.py | 47 ---- DemonstrationUniverse.cs | 66 ------ DemonstrationUniverse.py | 50 ----- DropboxDownloader.py | 119 ---------- MyCustomDataType.cs | 156 ------------- MyCustomDataUniverseType.cs | 141 ------------ QuantConnect.DataSource.csproj | 27 --- examples.md | 1 - output/alternative/mycustomdatatype/spy.csv | 6 - renameDataset.sh | 57 ----- tests/MyCustomDataTypeTests.cs | 99 -------- tests/Tests.csproj | 23 -- 21 files changed, 1343 deletions(-) delete mode 100644 DataProcessing/CLRImports.py delete mode 100644 DataProcessing/DataProcessing.csproj delete mode 100644 DataProcessing/MyCustomDataDownloader.cs delete mode 100644 DataProcessing/Program.cs delete mode 100644 DataProcessing/config.json delete mode 100644 DataProcessing/process.sample.ipynb delete mode 100644 DataProcessing/process.sample.py delete mode 100644 DataProcessing/process.sample.sh delete mode 100644 Demonstration.cs delete mode 100644 Demonstration.py delete mode 100644 DemonstrationUniverse.cs delete mode 100644 DemonstrationUniverse.py delete mode 100644 DropboxDownloader.py delete mode 100644 MyCustomDataType.cs delete mode 100644 MyCustomDataUniverseType.cs delete mode 100644 QuantConnect.DataSource.csproj delete mode 100644 examples.md delete mode 100644 output/alternative/mycustomdatatype/spy.csv delete mode 100644 renameDataset.sh delete mode 100644 tests/MyCustomDataTypeTests.cs delete mode 100644 tests/Tests.csproj diff --git a/DataProcessing/CLRImports.py b/DataProcessing/CLRImports.py deleted file mode 100644 index fca9342..0000000 --- a/DataProcessing/CLRImports.py +++ /dev/null @@ -1,12 +0,0 @@ -# This file is used to import the environment and classes/methods of LEAN. -# so that any python file could be using LEAN's classes/methods. -from clr_loader import get_coreclr -from pythonnet import set_runtime - -# process.runtimeconfig.json is created when we build the DataProcessing Project: -# dotnet build .\DataProcessing\DataProcessing.csproj -set_runtime(get_coreclr('process.runtimeconfig.json')) - -from AlgorithmImports import * -from QuantConnect.Lean.Engine.DataFeeds import * -AddReference("Fasterflect") \ No newline at end of file diff --git a/DataProcessing/DataProcessing.csproj b/DataProcessing/DataProcessing.csproj deleted file mode 100644 index 26b5466..0000000 --- a/DataProcessing/DataProcessing.csproj +++ /dev/null @@ -1,30 +0,0 @@ - - - Exe - net6.0 - process - true - - - - - - - - - - - - - - - PreserveNewest - - - - - - PreserveNewest - - - \ No newline at end of file diff --git a/DataProcessing/MyCustomDataDownloader.cs b/DataProcessing/MyCustomDataDownloader.cs deleted file mode 100644 index 1bfef32..0000000 --- a/DataProcessing/MyCustomDataDownloader.cs +++ /dev/null @@ -1,236 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json; -using QuantConnect.Configuration; -using QuantConnect.Data.Auxiliary; -using QuantConnect.DataSource; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Logging; -using QuantConnect.Util; - -namespace QuantConnect.DataProcessing -{ - /// - /// MyCustomDataDownloader implementation. - /// - public class MyCustomDataDownloader : IDisposable - { - public const string VendorName = "VendorName"; - public const string VendorDataName = "VendorDataName"; - - private readonly string _destinationFolder; - private readonly string _universeFolder; - private readonly string _clientKey; - private readonly string _dataFolder = Globals.DataFolder; - private readonly bool _canCreateUniverseFiles; - private readonly int _maxRetries = 5; - private static readonly List _defunctDelimiters = new() - { - '-', - '_' - }; - private ConcurrentDictionary> _tempData = new(); - - private readonly JsonSerializerSettings _jsonSerializerSettings = new() - { - DateTimeZoneHandling = DateTimeZoneHandling.Utc - }; - - /// - /// Control the rate of download per unit of time. - /// - private readonly RateGate _indexGate; - - /// - /// Creates a new instance of - /// - /// The folder where the data will be saved - /// The Vendor API key - public MyCustomDataDownloader(string destinationFolder, string apiKey = null) - { - _destinationFolder = Path.Combine(destinationFolder, VendorDataName); - _universeFolder = Path.Combine(_destinationFolder, "universe"); - _clientKey = apiKey ?? Config.Get("vendor-auth-token"); - _canCreateUniverseFiles = Directory.Exists(Path.Combine(_dataFolder, "equity", "usa", "map_files")); - - // Represents rate limits of 10 requests per 1.1 second - _indexGate = new RateGate(10, TimeSpan.FromSeconds(1.1)); - - Directory.CreateDirectory(_destinationFolder); - Directory.CreateDirectory(_universeFolder); - } - - /// - /// Runs the instance of the object. - /// - /// True if process all downloads successfully - public bool Run() - { - var stopwatch = Stopwatch.StartNew(); - var today = DateTime.UtcNow.Date; - - throw new NotImplementedException(); - - Log.Trace($"MyCustomDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); - return true; - } - - /// - /// Sends a GET request for the provided URL - /// - /// URL to send GET request for - /// Content as string - /// Failed to get data after exceeding retries - private async Task HttpRequester(string url) - { - for (var retries = 1; retries <= _maxRetries; retries++) - { - try - { - using (var client = new HttpClient()) - { - client.BaseAddress = new Uri(""); - client.DefaultRequestHeaders.Clear(); - - // You must supply your API key in the HTTP header, - // otherwise you will receive a 403 Forbidden response - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", _clientKey); - - // Responses are in JSON: you need to specify the HTTP header Accept: application/json - client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - - // Makes sure we don't overrun Quiver rate limits accidentally - _indexGate.WaitToProceed(); - - var response = await client.GetAsync(Uri.EscapeUriString(url)); - if (response.StatusCode == HttpStatusCode.NotFound) - { - Log.Error($"MyCustomDataDownloader.HttpRequester(): Files not found at url: {Uri.EscapeUriString(url)}"); - response.DisposeSafely(); - return string.Empty; - } - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - var finalRequestUri = response.RequestMessage.RequestUri; // contains the final location after following the redirect. - response = client.GetAsync(finalRequestUri).Result; // Reissue the request. The DefaultRequestHeaders configured on the client will be used, so we don't have to set them again. - } - - response.EnsureSuccessStatusCode(); - - var result = await response.Content.ReadAsStringAsync(); - response.DisposeSafely(); - - return result; - } - } - catch (Exception e) - { - Log.Error(e, $"MyCustomDataDownloader.HttpRequester(): Error at HttpRequester. (retry {retries}/{_maxRetries})"); - Thread.Sleep(1000); - } - } - - throw new Exception($"Request failed with no more retries remaining (retry {_maxRetries}/{_maxRetries})"); - } - - /// - /// Saves contents to disk, deleting existing zip files - /// - /// Final destination of the data - /// file name - /// Contents to write - private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) - { - name = name.ToLowerInvariant(); - var finalPath = Path.Combine(destinationFolder, $"{name}.csv"); - var finalFileExists = File.Exists(finalPath); - - var lines = new HashSet(contents); - if (finalFileExists) - { - foreach (var line in File.ReadAllLines(finalPath)) - { - lines.Add(line); - } - } - - var finalLines = destinationFolder.Contains("universe") ? - lines.OrderBy(x => x.Split(',').First()).ToList() : - lines - .OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)) - .ToList(); - - var tempPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.tmp"); - File.WriteAllLines(tempPath, finalLines); - var tempFilePath = new FileInfo(tempPath); - tempFilePath.MoveTo(finalPath, true); - } - - /// - /// Tries to normalize a potentially defunct ticker into a normal ticker. - /// - /// Ticker as received from Estimize - /// Set as the non-defunct ticker - /// true for success, false for failure - private static bool TryNormalizeDefunctTicker(string ticker, out string nonDefunctTicker) - { - // The "defunct" indicator can be in any capitalization/case - if (ticker.IndexOf("defunct", StringComparison.OrdinalIgnoreCase) > 0) - { - foreach (var delimChar in _defunctDelimiters) - { - var length = ticker.IndexOf(delimChar); - - // Continue until we exhaust all delimiters - if (length == -1) - { - continue; - } - - nonDefunctTicker = ticker[..length].Trim(); - return true; - } - - nonDefunctTicker = string.Empty; - return false; - } - - nonDefunctTicker = ticker; - return true; - } - - /// - /// Disposes of unmanaged resources - /// - public void Dispose() - { - _indexGate?.Dispose(); - } - } -} \ No newline at end of file diff --git a/DataProcessing/Program.cs b/DataProcessing/Program.cs deleted file mode 100644 index b203eb3..0000000 --- a/DataProcessing/Program.cs +++ /dev/null @@ -1,81 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.IO; -using QuantConnect.Configuration; -using QuantConnect.Logging; -using QuantConnect.Util; - -namespace QuantConnect.DataProcessing -{ - /// - /// Entrypoint for the data downloader/converter - /// - public class Program - { - /// - /// Entrypoint of the program - /// - /// Exit code. 0 equals successful, and any other value indicates the downloader/converter failed. - public static void Main() - { - // Get the config values first before running. These values are set for us - // automatically to the value set on the website when defining this data type - var destinationDirectory = Path.Combine( - Config.Get("temp-output-directory", "/temp-output-directory"), - "alternative", - "vendorname"); - - MyCustomDataDownloader instance = null; - try - { - // Pass in the values we got from the configuration into the downloader/converter. - instance = new MyCustomDataDownloader(destinationDirectory); - } - catch (Exception err) - { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {MyCustomDataDownloader.VendorDataName} {MyCustomDataDownloader.VendorDataName} data failed to be constructed"); - Environment.Exit(1); - } - - // No need to edit anything below here for most use cases. - // The downloader/converter is ran and cleaned up for you safely here. - try - { - // Run the data downloader/converter. - var success = instance.Run(); - if (!success) - { - Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {MyCustomDataDownloader.VendorName} {MyCustomDataDownloader.VendorDataName} data"); - Environment.Exit(1); - } - } - catch (Exception err) - { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {MyCustomDataDownloader.VendorDataName} {MyCustomDataDownloader.VendorDataName} data exited unexpectedly"); - Environment.Exit(1); - } - finally - { - // Run cleanup of the downloader/converter once it has finished or crashed. - instance.DisposeSafely(); - } - - // The downloader/converter was successful - Environment.Exit(0); - } - } -} \ No newline at end of file diff --git a/DataProcessing/config.json b/DataProcessing/config.json deleted file mode 100644 index 1d1e2f2..0000000 --- a/DataProcessing/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "data-folder": "../../../Data/", - - "vendor-auth-token": "" -} \ No newline at end of file diff --git a/DataProcessing/process.sample.ipynb b/DataProcessing/process.sample.ipynb deleted file mode 100644 index 2493379..0000000 --- a/DataProcessing/process.sample.ipynb +++ /dev/null @@ -1,78 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "9b8eae46", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# CLRImports is required to handle Lean C# objects for Mapped Datasets (Single asset and Universe Selection)\n", - "# Requirements:\n", - "# python -m pip install clr-loader==0.1.7\n", - "# python -m pip install pythonnet==3.0.0a2\n", - "# This script must be executed in ./bin/Debug/net6.0 after the follwing command is executed\n", - "# dotnet build .\\DataProcessing\\\n", - "import os\n", - "from CLRImports import *\n", - "\n", - "# To use QuantBook, we need to set its internal handlers\n", - "# We download LEAN confif with the default settings \n", - "with open(\"quantbook.json\", 'w') as fp:\n", - " from requests import get\n", - " response = get(\"https://raw.githubusercontent.com/QuantConnect/Lean/master/Launcher/config.json\")\n", - " fp.write(response.text)\n", - "\n", - "Config.SetConfigurationFile(\"quantbook.json\")\n", - "Config.Set(\"composer-dll-directory\", os.path.abspath(''))\n", - "\n", - "# Set the data folder\n", - "Config.Set(\"data-folder\", '')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6ddc2ed2-5690-422c-8c91-6e6f64dd45cb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# To generate the Security Identifier, we need to create and initialize the Map File Provider\n", - "# and call the SecurityIdentifier.GenerateEquity method\n", - "mapFileProvider = LocalZipMapFileProvider()\n", - "mapFileProvider.Initialize(DefaultDataProvider())\n", - "sid = SecurityIdentifier.GenerateEquity(\"SPY\", Market.USA, True, mapFileProvider, datetime(2022, 3, 1))\n", - "\n", - "qb = QuantBook()\n", - "symbol = Symbol(sid, \"SPY\")\n", - "history = qb.History(symbol, 3600, Resolution.Daily)\n", - "print(history)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/DataProcessing/process.sample.py b/DataProcessing/process.sample.py deleted file mode 100644 index 5b8285d..0000000 --- a/DataProcessing/process.sample.py +++ /dev/null @@ -1,32 +0,0 @@ -# CLRImports is required to handle Lean C# objects for Mapped Datasets (Single asset and Universe Selection) -# Requirements: -# python -m pip install clr-loader==0.1.7 -# python -m pip install pythonnet==3.0.0a2 -# This script must be executed in ./bin/Debug/net6.0 after the follwing command is executed -# dotnet build .\DataProcessing\ -import os -from CLRImports import * - -# To use QuantBook, we need to set its internal handlers -# We download LEAN confif with the default settings -with open("quantbook.json", 'w') as fp: - from requests import get - response = get("https://raw.githubusercontent.com/QuantConnect/Lean/master/Launcher/config.json") - fp.write(response.text) - -Config.SetConfigurationFile("quantbook.json") -Config.Set("composer-dll-directory", os.path.dirname(os.path.realpath(__file__))) - -# Set the data folder -Config.Set("data-folder", '') - -# To generate the Security Identifier, we need to create and initialize the Map File Provider -# and call the SecurityIdentifier.GenerateEquity method -mapFileProvider = LocalZipMapFileProvider() -mapFileProvider.Initialize(DefaultDataProvider()) -sid = SecurityIdentifier.GenerateEquity("SPY", Market.USA, True, mapFileProvider, datetime(2022, 3, 1)) - -qb = QuantBook() -symbol = Symbol(sid, "SPY") -history = qb.History(symbol, 3600, Resolution.Daily) -print(history) \ No newline at end of file diff --git a/DataProcessing/process.sample.sh b/DataProcessing/process.sample.sh deleted file mode 100644 index e69de29..0000000 diff --git a/Demonstration.cs b/Demonstration.cs deleted file mode 100644 index fc0eb1f..0000000 --- a/Demonstration.cs +++ /dev/null @@ -1,77 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using QuantConnect.Data; -using QuantConnect.Util; -using QuantConnect.Orders; -using QuantConnect.Algorithm; -using QuantConnect.DataSource; - -namespace QuantConnect.DataLibrary.Tests -{ - /// - /// Example algorithm using the custom data type as a source of alpha - /// - public class CustomDataAlgorithm : QCAlgorithm - { - private Symbol _customDataSymbol; - private Symbol _equitySymbol; - - /// - /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. - /// - public override void Initialize() - { - SetStartDate(2013, 10, 07); //Set Start Date - SetEndDate(2013, 10, 11); //Set End Date - _equitySymbol = AddEquity("SPY").Symbol; - _customDataSymbol = AddData(_equitySymbol).Symbol; - } - - /// - /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. - /// - /// Slice object keyed by symbol containing the stock data - public override void OnData(Slice slice) - { - var data = slice.Get(); - if (!data.IsNullOrEmpty()) - { - // based on the custom data property we will buy or short the underlying equity - if (data[_customDataSymbol].SomeCustomProperty == "buy") - { - SetHoldings(_equitySymbol, 1); - } - else if (data[_customDataSymbol].SomeCustomProperty == "sell") - { - SetHoldings(_equitySymbol, -1); - } - } - } - - /// - /// Order fill event handler. On an order fill update the resulting information is passed to this method. - /// - /// Order event details containing details of the events - public override void OnOrderEvent(OrderEvent orderEvent) - { - if (orderEvent.Status.IsFill()) - { - Debug($"Purchased Stock: {orderEvent.Symbol}"); - } - } - } -} diff --git a/Demonstration.py b/Demonstration.py deleted file mode 100644 index 2f55e0d..0000000 --- a/Demonstration.py +++ /dev/null @@ -1,47 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from AlgorithmImports import * - -### -### Example algorithm using the custom data type as a source of alpha -### -class CustomDataAlgorithm(QCAlgorithm): - def Initialize(self): - ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.''' - - self.SetStartDate(2020, 10, 7) #Set Start Date - self.SetEndDate(2020, 10, 11) #Set End Date - self.equity_symbol = self.AddEquity("SPY", Resolution.Daily).Symbol - self.custom_data_symbol = self.AddData(MyCustomDataType, self.equity_symbol).Symbol - - def OnData(self, slice): - ''' OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. - - :param Slice slice: Slice object keyed by symbol containing the stock data - ''' - data = slice.Get(MyCustomDataType) - if data: - custom_data = data[self.custom_data_symbol] - if custom_data.SomeCustomProperty == "buy": - self.SetHoldings(self.equitySymbol, 1) - elif custom_data.SomeCustomProperty == "sell": - self.SetHoldings(self.equitySymbol, -1) - - def OnOrderEvent(self, orderEvent): - ''' Order fill event handler. On an order fill update the resulting information is passed to this method. - - :param OrderEvent orderEvent: Order event details containing details of the events - ''' - if orderEvent.Status == OrderStatus.Fill: - self.Debug(f'Purchased Stock: {orderEvent.Symbol}') \ No newline at end of file diff --git a/DemonstrationUniverse.cs b/DemonstrationUniverse.cs deleted file mode 100644 index d8b962c..0000000 --- a/DemonstrationUniverse.cs +++ /dev/null @@ -1,66 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Linq; -using QuantConnect.Data; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.DataSource; - -namespace QuantConnect.Algorithm.CSharp -{ - /// - /// Example algorithm using the custom data type as a source of alpha - /// - public class CustomDataUniverse : QCAlgorithm - { - /// - /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. - /// - public override void Initialize() - { - // Data ADDED via universe selection is added with Daily resolution. - UniverseSettings.Resolution = Resolution.Daily; - - SetStartDate(2022, 2, 14); - SetEndDate(2022, 2, 18); - SetCash(100000); - - // add a custom universe data source (defaults to usa-equity) - AddUniverse("MyCustomDataUniverseType", Resolution.Daily, data => - { - foreach (var datum in data) - { - Log($"{datum.Symbol},{datum.SomeCustomProperty},{datum.SomeNumericProperty}"); - } - - // define our selection criteria - return from d in data - where d.SomeCustomProperty == "buy" - select d.Symbol; - }); - } - - /// - /// Event fired each time that we add/remove securities from the data feed - /// - /// Security additions/removals for this time step - public override void OnSecuritiesChanged(SecurityChanges changes) - { - Log(changes.ToString()); - } - } -} \ No newline at end of file diff --git a/DemonstrationUniverse.py b/DemonstrationUniverse.py deleted file mode 100644 index 80b3657..0000000 --- a/DemonstrationUniverse.py +++ /dev/null @@ -1,50 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from AlgorithmImports import * - -### -### Example algorithm using the custom data type as a source of alpha -### -class CustomDataUniverse(QCAlgorithm): - def Initialize(self): - ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. ''' - - # Data ADDED via universe selection is added with Daily resolution. - self.UniverseSettings.Resolution = Resolution.Daily - - self.SetStartDate(2022, 2, 14) - self.SetEndDate(2022, 2, 18) - self.SetCash(100000) - - # add a custom universe data source (defaults to usa-equity) - self.AddUniverse(MyCustomDataUniverseType, "MyCustomDataUniverseType", Resolution.Daily, self.UniverseSelection) - - def UniverseSelection(self, data): - ''' Selected the securities - - :param List of MyCustomUniverseType data: List of MyCustomUniverseType - :return: List of Symbol objects ''' - - for datum in data: - self.Log(f"{datum.Symbol},{datum.Followers},{datum.DayPercentChange},{datum.WeekPercentChange}") - - # define our selection criteria - return [d.Symbol for d in data if d.SomeCustomProperty == 'buy'] - - def OnSecuritiesChanged(self, changes): - ''' Event fired each time that we add/remove securities from the data feed - - :param SecurityChanges changes: Security additions/removals for this time step - ''' - self.Log(changes.ToString()) \ No newline at end of file diff --git a/DropboxDownloader.py b/DropboxDownloader.py deleted file mode 100644 index 54b7e55..0000000 --- a/DropboxDownloader.py +++ /dev/null @@ -1,119 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to download files from a given dropbox directory. -# Files to be downloaded are filtered based on given date present in file name. - -# ARGUMENTS -# DROPBOX_API_KEY: Dropbox API KEY with read access. -# DROPBOX_SOURCE_DIRECTORY: path of the dropbox directory to search files within. -# DROPBOX_OUTPUT_DIRECTORY(optional): base path of the output directory to store to downloaded files. -# cmdline args expected in order: DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - -import requests -import json -import sys -import time -import os -from pathlib import Path - -DROPBOX_API_KEY = os.environ.get("DROPBOX_API_KEY") -DROPBOX_SOURCE_DIRECTORY = os.environ.get("DROPBOX_SOURCE_DIRECTORY") -QC_DATAFLEET_DEPLOYMENT_DATE = os.environ.get("QC_DATAFLEET_DEPLOYMENT_DATE") -DROPBOX_OUTPUT_DIRECTORY = os.environ.get("DROPBOX_OUTPUT_DIRECTORY", "/raw") - -def DownloadZipFile(filePath): - - print(f"Starting downloading file at: {filePath}") - - # defining the api-endpoint - API_ENDPOINT_DOWNLOAD = "https://content.dropboxapi.com/2/files/download" - - # data to be sent to api - data = {"path": filePath} - - headers = {"Authorization": f"Bearer {DROPBOX_API_KEY}", - "Dropbox-API-Arg": json.dumps(data)} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_DOWNLOAD, headers=headers) - - response.raise_for_status() # ensure we notice bad responses - - fileName = filePath.split("/")[-1] - outputPath = os.path.join(DROPBOX_OUTPUT_DIRECTORY, fileName) - - with open(outputPath, "wb") as f: - f.write(response.content) - print(f"Succesfully saved file at: {outputPath}") - -def GetFilePathsFromDate(targetLocation, dateString): - # defining the api-endpoint - API_ENDPOINT_FILEPATH = "https://api.dropboxapi.com/2/files/list_folder" - - headers = {"Content-Type": "application/json", - "Authorization": f"Bearer {DROPBOX_API_KEY}"} - - # data to be sent to api - data = {"path": targetLocation, - "recursive": False, - "include_media_info": False, - "include_deleted": False, - "include_has_explicit_shared_members": False, - "include_mounted_folders": True, - "include_non_downloadable_files": True} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_FILEPATH, headers=headers, data = json.dumps(data)) - - response.raise_for_status() # ensure we notice bad responses - - target_paths = [entry["path_display"] for entry in response.json()["entries"] if dateString in entry["path_display"]] - return target_paths - -def main(): - global DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - inputCount = len(sys.argv) - if inputCount > 1: - DROPBOX_API_KEY = sys.argv[1] - if inputCount > 2: - DROPBOX_SOURCE_DIRECTORY = sys.argv[2] - if inputCount > 3: - QC_DATAFLEET_DEPLOYMENT_DATE = sys.argv[3] - if inputCount > 4: - DROPBOX_OUTPUT_DIRECTORY = sys.argv[4] - - # make output path if doesn't exists - Path(DROPBOX_OUTPUT_DIRECTORY).mkdir(parents=True, exist_ok=True) - - target_paths = GetFilePathsFromDate(DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE) - print(f"Found {len(target_paths)} files with following paths {target_paths}") - - #download files - for path in target_paths: - count = 0 - maxTries = 3 - while True: - try: - DownloadZipFile(path) - break - except Exception as e: - count +=1 - if count > maxTries: - print(f"Error for file with path {path} --error message: {e}") - break - print(f"Error, sleep for 5 sec and retry download file with --path: {path}") - time.sleep(5) - -if __name__== "__main__": - main() diff --git a/MyCustomDataType.cs b/MyCustomDataType.cs deleted file mode 100644 index 8c8a025..0000000 --- a/MyCustomDataType.cs +++ /dev/null @@ -1,156 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NodaTime; -using ProtoBuf; -using System.IO; -using QuantConnect.Data; -using System.Collections.Generic; - -namespace QuantConnect.DataSource -{ - /// - /// Example custom data type - /// - [ProtoContract(SkipConstructor = true)] - public class MyCustomDataType : BaseData - { - /// - /// Some custom data property - /// - [ProtoMember(2000)] - public string SomeCustomProperty { get; set; } - - /// - /// Time passed between the date of the data and the time the data became available to us - /// - public TimeSpan Period { get; set; } = TimeSpan.FromDays(1); - - /// - /// Time the data became available - /// - public override DateTime EndTime => Time + Period; - - /// - /// Return the URL string source of the file. This will be converted to a stream - /// - /// Configuration object - /// Date of this source file - /// true if we're in live mode, false for backtesting mode - /// String URL of source file. - public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) - { - return new SubscriptionDataSource( - Path.Combine( - Globals.DataFolder, - "alternative", - "mycustomdatatype", - $"{config.Symbol.Value.ToLowerInvariant()}.csv" - ), - SubscriptionTransportMedium.LocalFile - ); - } - - /// - /// Parses the data from the line provided and loads it into LEAN - /// - /// Subscription configuration - /// Line of data - /// Date - /// Is live mode - /// New instance - public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) - { - var csv = line.Split(','); - - var parsedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); - return new MyCustomDataType - { - Symbol = config.Symbol, - SomeCustomProperty = csv[1], - Time = parsedDate - Period, - }; - } - - /// - /// Clones the data - /// - /// A clone of the object - public override BaseData Clone() - { - return new MyCustomDataType - { - Symbol = Symbol, - Time = Time, - EndTime = EndTime, - SomeCustomProperty = SomeCustomProperty, - }; - } - - /// - /// Indicates whether the data source is tied to an underlying symbol and requires that corporate events be applied to it as well, such as renames and delistings - /// - /// false - public override bool RequiresMapping() - { - return true; - } - - /// - /// Indicates whether the data is sparse. - /// If true, we disable logging for missing files - /// - /// true - public override bool IsSparseData() - { - return true; - } - - /// - /// Converts the instance to string - /// - public override string ToString() - { - return $"{Symbol} - {SomeCustomProperty}"; - } - - /// - /// Gets the default resolution for this data and security type - /// - public override Resolution DefaultResolution() - { - return Resolution.Daily; - } - - /// - /// Gets the supported resolution for this data and security type - /// - public override List SupportedResolutions() - { - return DailyResolution; - } - - /// - /// Specifies the data time zone for this data type. This is useful for custom data types - /// - /// The of this data type - public override DateTimeZone DataTimeZone() - { - return DateTimeZone.Utc; - } - } -} diff --git a/MyCustomDataUniverseType.cs b/MyCustomDataUniverseType.cs deleted file mode 100644 index 5db2f4b..0000000 --- a/MyCustomDataUniverseType.cs +++ /dev/null @@ -1,141 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NodaTime; -using ProtoBuf; -using System.IO; -using QuantConnect.Data; -using System.Collections.Generic; -using System.Globalization; - -namespace QuantConnect.DataSource -{ - /// - /// Example custom data type - /// - [ProtoContract(SkipConstructor = true)] - public class MyCustomDataUniverseType : BaseData - { - /// - /// Some custom data property - /// - public string SomeCustomProperty { get; set; } - - /// - /// Some custom data property - /// - public decimal SomeNumericProperty { get; set; } - - /// - /// Time passed between the date of the data and the time the data became available to us - /// - public TimeSpan Period { get; set; } = TimeSpan.FromDays(1); - - /// - /// Time the data became available - /// - public override DateTime EndTime => Time + Period; - - /// - /// Return the URL string source of the file. This will be converted to a stream - /// - /// Configuration object - /// Date of this source file - /// true if we're in live mode, false for backtesting mode - /// String URL of source file. - public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) - { - return new SubscriptionDataSource( - Path.Combine( - Globals.DataFolder, - "alternative", - "mycustomdatatype", - "universe", - $"{date.ToStringInvariant(DateFormat.EightCharacter)}.csv" - ), - SubscriptionTransportMedium.LocalFile - ); - } - - /// - /// Parses the data from the line provided and loads it into LEAN - /// - /// Subscription configuration - /// Line of data - /// Date - /// Is live mode - /// New instance - public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) - { - var csv = line.Split(','); - - var someNumericProperty = decimal.Parse(csv[2], NumberStyles.Any, CultureInfo.InvariantCulture); - - return new MyCustomDataUniverseType - { - Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]), - SomeNumericProperty = someNumericProperty, - SomeCustomProperty = csv[3], - Time = date - Period, - Value = someNumericProperty - }; - } - - /// - /// Indicates whether the data is sparse. - /// If true, we disable logging for missing files - /// - /// true - public override bool IsSparseData() - { - return true; - } - - /// - /// Converts the instance to string - /// - public override string ToString() - { - return $"{Symbol} - {Value}"; - } - - /// - /// Gets the default resolution for this data and security type - /// - public override Resolution DefaultResolution() - { - return Resolution.Daily; - } - - /// - /// Gets the supported resolution for this data and security type - /// - public override List SupportedResolutions() - { - return DailyResolution; - } - - /// - /// Specifies the data time zone for this data type. This is useful for custom data types - /// - /// The of this data type - public override DateTimeZone DataTimeZone() - { - return DateTimeZone.Utc; - } - } -} \ No newline at end of file diff --git a/QuantConnect.DataSource.csproj b/QuantConnect.DataSource.csproj deleted file mode 100644 index f379576..0000000 --- a/QuantConnect.DataSource.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - net6.0 - QuantConnect.DataSource - QuantConnect.DataSource.MyCustomDataType - bin\$(Configuration) - $(OutputPath)\QuantConnect.DataSource.MyCustomDataType.xml - - - - - - - - - - - - - - - - - - - - diff --git a/examples.md b/examples.md deleted file mode 100644 index 9086ae3..0000000 --- a/examples.md +++ /dev/null @@ -1 +0,0 @@ -https://github.com/QuantConnect?q=Lean.DataSource&type=&language=&sort= \ No newline at end of file diff --git a/output/alternative/mycustomdatatype/spy.csv b/output/alternative/mycustomdatatype/spy.csv deleted file mode 100644 index e450a4d..0000000 --- a/output/alternative/mycustomdatatype/spy.csv +++ /dev/null @@ -1,6 +0,0 @@ -20131001,buy -20131003,buy -20131006,buy -20131007,sell -20131009,buy -20131011,sell \ No newline at end of file diff --git a/renameDataset.sh b/renameDataset.sh deleted file mode 100644 index 51ccd6d..0000000 --- a/renameDataset.sh +++ /dev/null @@ -1,57 +0,0 @@ -# Get {vendorNameDatasetName} -vendorNameDatasetName=${PWD##*.} -vendorNameDatasetNameUniverse=${vendorNameDatasetName}Universe - -# Rename the MyCustomDataType.cs file to {vendorNameDatasetName}.cs -mv MyCustomDataType.cs ${vendorNameDatasetName}.cs -mv MyCustomDataUniverseType.cs ${vendorNameDatasetNameUniverse}.cs - -# In the QuantConnect.DataSource.csproj file, rename the MyCustomDataType class to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" QuantConnect.DataSource.csproj -sed -i "s/Demonstration.cs/${vendorNameDatasetName}Algorithm.cs/g" QuantConnect.DataSource.csproj -sed -i "s/DemonstrationUniverse.cs/${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs/g" QuantConnect.DataSource.csproj - -# In the {vendorNameDatasetName}.cs file, rename the MyCustomDataType class to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" ${vendorNameDatasetName}.cs - -# In the {vendorNameDatasetNameUniverse}.cs file, rename the MyCustomDataUniverseType class to {vendorNameDatasetNameUniverse} -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" ${vendorNameDatasetNameUniverse}.cs - -# In the {vendorNameDatasetName}Algorithm.cs file, rename the MyCustomDataType class to to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" Demonstration.cs -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" Demonstration.py - -# In the {vendorNameDatasetName}Algorithm.cs file, rename the CustomDataAlgorithm class to {vendorNameDatasetName}Algorithm -sed -i "s/CustomDataAlgorithm/${vendorNameDatasetName}Algorithm/g" Demonstration.cs -sed -i "s/CustomDataAlgorithm/${vendorNameDatasetName}Algorithm/g" Demonstration.py - -# In the {vendorNameDatasetName}UniverseSelectionAlgorithm.cs file, rename the MyCustomDataUniverseType class to to {vendorNameDatasetName}Universe -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" DemonstrationUniverse.cs -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" DemonstrationUniverse.py - -# In the {vendorNameDatasetNameUniverse}SelectionAlgorithm.cs file, rename the CustomDataAlgorithm class to {vendorNameDatasetNameUniverse}SelectionAlgorithm -sed -i "s/CustomDataUniverse/${vendorNameDatasetNameUniverse}SelectionAlgorithm/g" DemonstrationUniverse.cs -sed -i "s/CustomDataUniverse/${vendorNameDatasetNameUniverse}SelectionAlgorithm/g" DemonstrationUniverse.py - -# Rename the Lean.DataSource.vendorNameDatasetName/Demonstration.cs/py file to {vendorNameDatasetName}Algorithm.cs/py -mv Demonstration.cs ${vendorNameDatasetName}Algorithm.cs -mv Demonstration.py ${vendorNameDatasetName}Algorithm.py - -# Rename the Lean.DataSource.vendorNameDatasetName/DemonstrationUniverseSelectionAlgorithm.cs/py file to {vendorNameDatasetName}UniverseSelectionAlgorithm.cs/py -mv DemonstrationUniverse.cs ${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs -mv DemonstrationUniverse.py ${vendorNameDatasetNameUniverse}SelectionAlgorithm.py - -# Rename the tests/MyCustomDataTypeTests.cs file to tests/{vendorNameDatasetName}Tests.cs -sed -i "s/MyCustomDataType/${vendorNameDatasetName}/g" tests/MyCustomDataTypeTests.cs -mv tests/MyCustomDataTypeTests.cs tests/${vendorNameDatasetName}Tests.cs - -# In tests/Tests.csproj, rename the Demonstration.cs and DemonstrationUniverse.cs to {vendorNameDatasetName}Algorithm.cs and {vendorNameDatasetNameUniverse}SelectionAlgorithm.cs -sed -i "s/Demonstration.cs/${vendorNameDatasetName}Algorithm.cs/g" tests/Tests.csproj -sed -i "s/DemonstrationUniverse.cs/${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs/g" tests/Tests.csproj - -# In the MyCustomDataDownloader.cs and Program.cs files, rename the MyCustomDataDownloader to {vendorNameDatasetNameUniverse}DataDownloader -sed -i "s/MyCustomDataDownloader/${vendorNameDatasetNameUniverse}DataDownloader/g" DataProcessing/Program.cs -sed -i "s/MyCustomDataDownloader/${vendorNameDatasetNameUniverse}DataDownloader/g" DataProcessing/MyCustomDataDownloader.cs - -# Rename the DataProcessing/MyCustomDataDownloader.cs file to DataProcessing/{vendorNameDatasetName}DataDownloader.cs -mv DataProcessing/MyCustomDataDownloader.cs DataProcessing/${vendorNameDatasetName}DataDownloader.cs \ No newline at end of file diff --git a/tests/MyCustomDataTypeTests.cs b/tests/MyCustomDataTypeTests.cs deleted file mode 100644 index c0b907c..0000000 --- a/tests/MyCustomDataTypeTests.cs +++ /dev/null @@ -1,99 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using ProtoBuf; -using System.IO; -using System.Linq; -using ProtoBuf.Meta; -using Newtonsoft.Json; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.DataSource; - -namespace QuantConnect.DataLibrary.Tests -{ - [TestFixture] - public class MyCustomDataTypeTests - { - [Test] - public void JsonRoundTrip() - { - var expected = CreateNewInstance(); - var type = expected.GetType(); - var serialized = JsonConvert.SerializeObject(expected); - var result = JsonConvert.DeserializeObject(serialized, type); - - AssertAreEqual(expected, result); - } - - [Test] - public void ProtobufRoundTrip() - { - var expected = CreateNewInstance(); - var type = expected.GetType(); - - RuntimeTypeModel.Default[typeof(BaseData)].AddSubType(2000, type); - - using (var stream = new MemoryStream()) - { - Serializer.Serialize(stream, expected); - - stream.Position = 0; - - var result = Serializer.Deserialize(type, stream); - - AssertAreEqual(expected, result, filterByCustomAttributes: true); - } - } - - [Test] - public void Clone() - { - var expected = CreateNewInstance(); - var result = expected.Clone(); - - AssertAreEqual(expected, result); - } - - private void AssertAreEqual(object expected, object result, bool filterByCustomAttributes = false) - { - foreach (var propertyInfo in expected.GetType().GetProperties()) - { - // we skip Symbol which isn't protobuffed - if (filterByCustomAttributes && propertyInfo.CustomAttributes.Count() != 0) - { - Assert.AreEqual(propertyInfo.GetValue(expected), propertyInfo.GetValue(result)); - } - } - foreach (var fieldInfo in expected.GetType().GetFields()) - { - Assert.AreEqual(fieldInfo.GetValue(expected), fieldInfo.GetValue(result)); - } - } - - private BaseData CreateNewInstance() - { - return new MyCustomDataType - { - Symbol = Symbol.Empty, - Time = DateTime.Today, - DataType = MarketDataType.Base, - SomeCustomProperty = "This is some market related information" - }; - } - } -} \ No newline at end of file diff --git a/tests/Tests.csproj b/tests/Tests.csproj deleted file mode 100644 index 8e308d0..0000000 --- a/tests/Tests.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - net6.0 - QuantConnect.DataLibrary.Tests - - - - - - - - - - all - - - - - - - - - From efa7454c478f9c95ea4cf76cf6912a87e0ae2476 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 8 Feb 2024 01:03:42 +0200 Subject: [PATCH 02/34] feat: prepare solution's structure feat: github issue & PR templates feat: prepare GH workflow file --- .github/ISSUE_TEMPLATE.md | 33 +++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 31 ++++++++++++++ .github/workflows/build.yml | 35 ++++++++++------ Lean.DataSource.CoinAPI.sln | 37 ++++++++++++++++ QuantConnect.CoinAPI.Converter/Program.cs | 2 + .../QuantConnect.CoinAPI.Converter.csproj | 35 ++++++++++++++++ .../QuantConnect.CoinAPI.Tests.csproj | 42 +++++++++++++++++++ QuantConnect.CoinAPI.Tests/config.json | 7 ++++ .../QuantConnect.CoinAPI.csproj | 39 +++++++++++++++++ 9 files changed, 248 insertions(+), 13 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 Lean.DataSource.CoinAPI.sln create mode 100644 QuantConnect.CoinAPI.Converter/Program.cs create mode 100644 QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj create mode 100644 QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj create mode 100644 QuantConnect.CoinAPI.Tests/config.json create mode 100644 QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..b1be05d --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,33 @@ + + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + +## Steps to Reproduce (for bugs) + + +1. +2. +3. +4. + +## Context + + + +## Your Environment + +* Version used: +* Environment name and version (e.g. PHP 5.4 on nginx 1.9.1): +* Server type and version: +* Operating System and version: +* Link to your project: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..4f9ceac --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,31 @@ + + +## Description + + +## Motivation and Context + + + +## How Has This Been Tested? + + + + +## Screenshots (if appropriate): + +## Types of changes + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to change) + +## Checklist: + + +- [ ] My code follows the code style of this project. +- [ ] My change requires a change to the documentation. +- [ ] I have updated the documentation accordingly. +- [ ] I have read the **CONTRIBUTING** document. +- [ ] I have added tests to cover my changes. +- [ ] All new and existing tests passed. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 09449fc..cd07fab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,18 +9,18 @@ on: jobs: build: runs-on: ubuntu-20.04 - + env: + QC_JOB_USER_ID: ${{ secrets.QC_JOB_USER_ID }} + QC_API_ACCESS_TOKEN: ${{ secrets.QC_API_ACCESS_TOKEN }} + QC_JOB_ORGANIZATION_ID: ${{ secrets.QC_JOB_ORGANIZATION_ID }} + QC_COINAPI_API_KEY: ${{ secrets.QC_COINAPI_API_KEY }} steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Free space run: df -h && rm -rf /opt/hostedtoolcache* && df -h - - name: Pull Foundation Image - uses: addnab/docker-run-action@v3 - with: - image: quantconnect/lean:foundation - - name: Checkout Lean Same Branch id: lean-same-branch uses: actions/checkout@v2 @@ -40,11 +40,20 @@ jobs: - name: Move Lean run: mv Lean ../Lean - - name: BuildDataSource - run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Pull Foundation Image + uses: addnab/docker-run-action@v3 + with: + image: quantconnect/lean:foundation + options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.CoinAPI/Lean.DataSource.CoinAPI -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_COINAPI_API_KEY=${{ secrets.QC_COINAPI_API_KEY }} + + - name: Build QuantConnect.CoinAPI + run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + + - name: Build QuantConnect.CoinAPI.Converter + run: dotnet build ./QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: BuildTests - run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Build QuantConnect.CoinAPI.Tests + run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: Run Tests - run: dotnet test ./tests/bin/Release/net6.0/Tests.dll + - name: Run QuantConnect.CoinAPI.Tests + run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.CoinAPI.Tests.dll \ No newline at end of file diff --git a/Lean.DataSource.CoinAPI.sln b/Lean.DataSource.CoinAPI.sln new file mode 100644 index 0000000..bdf8f8e --- /dev/null +++ b/Lean.DataSource.CoinAPI.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Converter", "QuantConnect.CoinAPI.Converter\QuantConnect.CoinAPI.Converter.csproj", "{881514B4-641E-4EDC-8020-6BEA0CC8F48C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Release|Any CPU.Build.0 = Release|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Release|Any CPU.Build.0 = Release|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {2C7F3105-1213-40AC-BEB7-3B5637C33F5D} + EndGlobalSection +EndGlobal diff --git a/QuantConnect.CoinAPI.Converter/Program.cs b/QuantConnect.CoinAPI.Converter/Program.cs new file mode 100644 index 0000000..3751555 --- /dev/null +++ b/QuantConnect.CoinAPI.Converter/Program.cs @@ -0,0 +1,2 @@ +// See https://aka.ms/new-console-template for more information +Console.WriteLine("Hello, World!"); diff --git a/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj b/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj new file mode 100644 index 0000000..8f25f33 --- /dev/null +++ b/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj @@ -0,0 +1,35 @@ + + + + Release + AnyCPU + net6.0 + QuantConnect.CoinAPI.Converter + QuantConnect.CoinAPI.Converter + QuantConnect.CoinAPI.Converter + QuantConnect.CoinAPI.Converter + Exe + bin\$(Configuration) + false + true + false + QuantConnect LEAN CoinAPI Converter Data Source: CoinAPI Converter Data Source plugin for Lean + enable + enable + + + + full + bin\Debug\ + + + + pdbonly + bin\Release\ + + + + + + + diff --git a/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj b/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj new file mode 100644 index 0000000..a46f7dd --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj @@ -0,0 +1,42 @@ + + + + Release + AnyCPU + net6.0 + bin\$(Configuration)\ + QuantConnect.CoinAPI.Tests + QuantConnect.CoinAPI.Tests + QuantConnect.CoinAPI.Tests + QuantConnect.CoinAPI.Tests + false + enable + enable + + false + true + UnitTest + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + PreserveNewest + + + + diff --git a/QuantConnect.CoinAPI.Tests/config.json b/QuantConnect.CoinAPI.Tests/config.json new file mode 100644 index 0000000..6241a29 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/config.json @@ -0,0 +1,7 @@ +{ + "data-folder": "../../../../Lean/Data/", + "data-directory": "../../../../Lean/Data/", + + "coinapi-api-key": "", + "coinapi-product": "free" +} diff --git a/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj b/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj new file mode 100644 index 0000000..cdd3a6d --- /dev/null +++ b/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj @@ -0,0 +1,39 @@ + + + + Release + AnyCPU + net6.0 + QuantConnect.CoinAPI + QuantConnect.CoinAPI + QuantConnect.CoinAPI + QuantConnect.CoinAPI + Library + bin\$(Configuration) + false + true + false + QuantConnect LEAN CoinAPI Data Source: CoinAPI Data Source plugin for Lean + enable + enable + + + + full + bin\Debug\ + + + + pdbonly + bin\Release\ + + + + + + + + + + + From aa864997ae002f98c3461036477478b4a4e8faa0 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 8 Feb 2024 02:17:21 +0200 Subject: [PATCH 03/34] feat: paste coinApi files from ToolBox --- .../CoinApiDataQueueHandler.cs | 507 ++++++++++++++++++ QuantConnect.CoinAPI/CoinApiProduct.cs | 49 ++ QuantConnect.CoinAPI/CoinApiSymbol.cs | 42 ++ QuantConnect.CoinAPI/CoinApiSymbolMapper.cs | 256 +++++++++ QuantConnect.CoinAPI/Messages/BaseMessage.cs | 25 + QuantConnect.CoinAPI/Messages/ErrorMessage.cs | 25 + QuantConnect.CoinAPI/Messages/HelloMessage.cs | 40 ++ .../Messages/HistoricalDataMessage.cs | 46 ++ QuantConnect.CoinAPI/Messages/QuoteMessage.cs | 46 ++ QuantConnect.CoinAPI/Messages/TradeMessage.cs | 46 ++ .../QuantConnect.CoinAPI.csproj | 3 +- 11 files changed, 1084 insertions(+), 1 deletion(-) create mode 100644 QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs create mode 100644 QuantConnect.CoinAPI/CoinApiProduct.cs create mode 100644 QuantConnect.CoinAPI/CoinApiSymbol.cs create mode 100644 QuantConnect.CoinAPI/CoinApiSymbolMapper.cs create mode 100644 QuantConnect.CoinAPI/Messages/BaseMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/ErrorMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/HelloMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/QuoteMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/TradeMessage.cs diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs new file mode 100644 index 0000000..fe38c04 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -0,0 +1,507 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NodaTime; +using RestSharp; +using Newtonsoft.Json; +using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Packets; +using QuantConnect.Logging; +using CoinAPI.WebSocket.V1; +using QuantConnect.Interfaces; +using QuantConnect.Data.Market; +using QuantConnect.Configuration; +using System.Collections.Concurrent; +using QuantConnect.CoinAPI.Messages; +using CoinAPI.WebSocket.V1.DataModels; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; +using HistoryRequest = QuantConnect.Data.HistoryRequest; + +namespace QuantConnect.CoinAPI +{ + /// + /// An implementation of for CoinAPI + /// + public class CoinApiDataQueueHandler : SynchronizingHistoryProvider, IDataQueueHandler + { + protected int HistoricalDataPerRequestLimit = 10000; + private static readonly Dictionary _ResolutionToCoinApiPeriodMappings = new Dictionary + { + { Resolution.Second, "1SEC"}, + { Resolution.Minute, "1MIN" }, + { Resolution.Hour, "1HRS" }, + { Resolution.Daily, "1DAY" }, + }; + + private readonly string _apiKey = Config.Get("coinapi-api-key"); + private readonly string[] _streamingDataType; + private readonly CoinApiWsClient _client; + private readonly object _locker = new object(); + private ConcurrentDictionary _symbolCache = new ConcurrentDictionary(); + private readonly CoinApiSymbolMapper _symbolMapper = new CoinApiSymbolMapper(); + private readonly IDataAggregator _dataAggregator; + private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; + + private readonly TimeSpan _subscribeDelay = TimeSpan.FromMilliseconds(250); + private readonly object _lockerSubscriptions = new object(); + private DateTime _lastSubscribeRequestUtcTime = DateTime.MinValue; + private bool _subscriptionsPending; + + private readonly TimeSpan _minimumTimeBetweenHelloMessages = TimeSpan.FromSeconds(5); + private DateTime _nextHelloMessageUtcTime = DateTime.MinValue; + + private readonly ConcurrentDictionary _previousQuotes = new ConcurrentDictionary(); + + /// + /// Initializes a new instance of the class + /// + public CoinApiDataQueueHandler() + { + _dataAggregator = Composer.Instance.GetPart(); + if (_dataAggregator == null) + { + _dataAggregator = + Composer.Instance.GetExportedValueByTypeName(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")); + } + var product = Config.GetValue("coinapi-product"); + _streamingDataType = product < CoinApiProduct.Streamer + ? new[] { "trade" } + : new[] { "trade", "quote" }; + + Log.Trace($"CoinApiDataQueueHandler(): using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); + + _client = new CoinApiWsClient(); + _client.TradeEvent += OnTrade; + _client.QuoteEvent += OnQuote; + _client.Error += OnError; + _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); + _subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s); + _subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s); + } + + /// + /// Subscribe to the specified configuration + /// + /// defines the parameters to subscribe to a data feed + /// handler to be fired on new data available + /// The new enumerator for this subscription request + public IEnumerator Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + { + if (!CanSubscribe(dataConfig.Symbol)) + { + return null; + } + + var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); + _subscriptionManager.Subscribe(dataConfig); + + return enumerator; + } + + /// + /// Sets the job we're subscribing for + /// + /// Job we're subscribing for + public void SetJob(LiveNodePacket job) + { + } + + /// + /// Adds the specified symbols to the subscription + /// + /// The symbols to be added keyed by SecurityType + private bool Subscribe(IEnumerable symbols) + { + ProcessSubscriptionRequest(); + return true; + } + + /// + /// Removes the specified configuration + /// + /// Subscription config to be removed + public void Unsubscribe(SubscriptionDataConfig dataConfig) + { + _subscriptionManager.Unsubscribe(dataConfig); + _dataAggregator.Remove(dataConfig); + } + + + /// + /// Removes the specified symbols to the subscription + /// + /// The symbols to be removed keyed by SecurityType + private bool Unsubscribe(IEnumerable symbols) + { + ProcessSubscriptionRequest(); + return true; + } + + /// + /// Returns whether the data provider is connected + /// + /// true if the data provider is connected + public bool IsConnected => true; + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() + { + _client.TradeEvent -= OnTrade; + _client.QuoteEvent -= OnQuote; + _client.Error -= OnError; + _client.Dispose(); + _dataAggregator.DisposeSafely(); + } + + /// + /// Helper method used in QC backend + /// + /// List of LEAN markets (exchanges) to subscribe + public void SubscribeMarkets(List markets) + { + Log.Trace($"CoinApiDataQueueHandler.SubscribeMarkets(): {string.Join(",", markets)}"); + + // we add '_' to be more precise, for example requesting 'BINANCE' doesn't match 'BINANCEUS' + SendHelloMessage(markets.Select(x => string.Concat(_symbolMapper.GetExchangeId(x.ToLowerInvariant()), "_"))); + } + + private void ProcessSubscriptionRequest() + { + if (_subscriptionsPending) return; + + _lastSubscribeRequestUtcTime = DateTime.UtcNow; + _subscriptionsPending = true; + + Task.Run(async () => + { + while (true) + { + DateTime requestTime; + List symbolsToSubscribe; + lock (_lockerSubscriptions) + { + requestTime = _lastSubscribeRequestUtcTime.Add(_subscribeDelay); + + // CoinAPI requires at least 5 seconds between hello messages + if (_nextHelloMessageUtcTime != DateTime.MinValue && requestTime < _nextHelloMessageUtcTime) + { + requestTime = _nextHelloMessageUtcTime; + } + + symbolsToSubscribe = _subscriptionManager.GetSubscribedSymbols().ToList(); + } + + var timeToWait = requestTime - DateTime.UtcNow; + + int delayMilliseconds; + if (timeToWait <= TimeSpan.Zero) + { + // minimum delay has passed since last subscribe request, send the Hello message + SubscribeSymbols(symbolsToSubscribe); + + lock (_lockerSubscriptions) + { + _lastSubscribeRequestUtcTime = DateTime.UtcNow; + if (_subscriptionManager.GetSubscribedSymbols().Count() == symbolsToSubscribe.Count) + { + // no more subscriptions pending, task finished + _subscriptionsPending = false; + break; + } + } + + delayMilliseconds = _subscribeDelay.Milliseconds; + } + else + { + delayMilliseconds = timeToWait.Milliseconds; + } + + await Task.Delay(delayMilliseconds).ConfigureAwait(false); + } + }); + } + + /// + /// Returns true if we can subscribe to the specified symbol + /// + private static bool CanSubscribe(Symbol symbol) + { + // ignore unsupported security types + if (symbol.ID.SecurityType != SecurityType.Crypto && symbol.ID.SecurityType != SecurityType.CryptoFuture) + { + return false; + } + + // ignore universe symbols + return !symbol.Value.Contains("-UNIVERSE-"); + } + + /// + /// Subscribes to a list of symbols + /// + /// The list of symbols to subscribe + private void SubscribeSymbols(List symbolsToSubscribe) + { + Log.Trace($"CoinApiDataQueueHandler.SubscribeSymbols(): {string.Join(",", symbolsToSubscribe)}"); + + // subscribe to symbols using exact match + SendHelloMessage(symbolsToSubscribe.Select(x => + { + try + { + var result = string.Concat(_symbolMapper.GetBrokerageSymbol(x), "$"); + return result; + } + catch (Exception e) + { + Log.Error(e); + return null; + } + }).Where(x => x != null)); + } + + private void SendHelloMessage(IEnumerable subscribeFilter) + { + var list = subscribeFilter.ToList(); + if (list.Count == 0) + { + // If we use a null or empty filter in the CoinAPI hello message + // we will be subscribing to all symbols for all active exchanges! + // Only option is requesting an invalid symbol as filter. + list.Add("$no_symbol_requested$"); + } + + _client.SendHelloMessage(new Hello + { + apikey = Guid.Parse(_apiKey), + heartbeat = true, + subscribe_data_type = _streamingDataType, + subscribe_filter_symbol_id = list.ToArray() + }); + + _nextHelloMessageUtcTime = DateTime.UtcNow.Add(_minimumTimeBetweenHelloMessages); + } + + private void OnTrade(object sender, Trade trade) + { + try + { + var symbol = GetSymbolUsingCache(trade.symbol_id); + if (symbol == null) + { + return; + } + + var tick = new Tick(trade.time_exchange, symbol, string.Empty, string.Empty, quantity: trade.size, price: trade.price); + + lock (symbol) + { + _dataAggregator.Update(tick); + } + } + catch (Exception e) + { + Log.Error(e); + } + } + + private void OnQuote(object sender, Quote quote) + { + try + { + // only emit quote ticks if bid price or ask price changed + Tick previousQuote; + if (!_previousQuotes.TryGetValue(quote.symbol_id, out previousQuote) + || quote.ask_price != previousQuote.AskPrice + || quote.bid_price != previousQuote.BidPrice) + { + var symbol = GetSymbolUsingCache(quote.symbol_id); + if (symbol == null) + { + return; + } + + var tick = new Tick(quote.time_exchange, symbol, string.Empty, string.Empty, + bidSize: quote.bid_size, bidPrice: quote.bid_price, + askSize: quote.ask_size, askPrice: quote.ask_price); + + _previousQuotes[quote.symbol_id] = tick; + lock (symbol) + { + _dataAggregator.Update(tick); + } + } + } + catch (Exception e) + { + Log.Error(e); + } + } + + private Symbol GetSymbolUsingCache(string ticker) + { + if (!_symbolCache.TryGetValue(ticker, out Symbol result)) + { + try + { + var securityType = ticker.IndexOf("_PERP_") > 0 ? SecurityType.CryptoFuture : SecurityType.Crypto; + result = _symbolMapper.GetLeanSymbol(ticker, securityType, string.Empty); + } + catch (Exception e) + { + Log.Error(e); + // we store the null so we don't keep going into the same mapping error + result = null; + } + _symbolCache[ticker] = result; + } + return result; + } + + private void OnError(object sender, Exception e) + { + Log.Error(e); + } + + #region SynchronizingHistoryProvider + + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + // NOP + } + + public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + var subscriptions = new List(); + foreach (var request in requests) + { + var history = GetHistory(request); + var subscription = CreateSubscription(request, history); + subscriptions.Add(subscription); + } + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + } + + public IEnumerable GetHistory(HistoryRequest historyRequest) + { + if (historyRequest.Symbol.SecurityType != SecurityType.Crypto && historyRequest.Symbol.SecurityType != SecurityType.CryptoFuture) + { + Log.Error($"CoinApiDataQueueHandler.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); + yield break; + } + + if (historyRequest.Resolution == Resolution.Tick) + { + Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical ticks, only OHLCV timeseries"); + yield break; + } + + if (historyRequest.DataType == typeof(QuoteBar)) + { + Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical QuoteBars , only TradeBars"); + yield break; + } + + var resolutionTimeSpan = historyRequest.Resolution.ToTimeSpan(); + var lastRequestedBarStartTime = historyRequest.EndTimeUtc.RoundDown(resolutionTimeSpan); + var currentStartTime = historyRequest.StartTimeUtc.RoundUp(resolutionTimeSpan); + var currentEndTime = lastRequestedBarStartTime; + + // Perform a check of the number of bars requested, this must not exceed a static limit + var dataRequestedCount = (currentEndTime - currentStartTime).Ticks + / resolutionTimeSpan.Ticks; + + if (dataRequestedCount > HistoricalDataPerRequestLimit) + { + currentEndTime = currentStartTime + + TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); + } + + while (currentStartTime < lastRequestedBarStartTime) + { + var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[historyRequest.Resolution]; + + // Time must be in ISO 8601 format + var coinApiStartTime = currentStartTime.ToStringInvariant("s"); + var coinApiEndTime = currentEndTime.ToStringInvariant("s"); + + // Construct URL for rest request + var baseUrl = + "https://rest.coinapi.io/v1/ohlcv/" + + $"{coinApiSymbol}/history?period_id={coinApiPeriod}&limit={HistoricalDataPerRequestLimit}" + + $"&time_start={coinApiStartTime}&time_end={coinApiEndTime}"; + + // Execute + var client = new RestClient(baseUrl); + var restRequest = new RestRequest(Method.GET); + restRequest.AddHeader("X-CoinAPI-Key", _apiKey); + var response = client.Execute(restRequest); + + // Log the information associated with the API Key's rest call limits. + TraceRestUsage(response); + + // Deserialize to array + var coinApiHistoryBars = JsonConvert.DeserializeObject(response.Content); + + // Can be no historical data for a short period interval + if (!coinApiHistoryBars.Any()) + { + Log.Error($"CoinApiDataQueueHandler.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]"); + continue; + } + + foreach (var ohlcv in coinApiHistoryBars) + { + yield return + new TradeBar(ohlcv.TimePeriodStart, historyRequest.Symbol, ohlcv.PriceOpen, ohlcv.PriceHigh, + ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, historyRequest.Resolution.ToTimeSpan()); + } + + currentStartTime = currentEndTime; + currentEndTime += TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); + } + } + + #endregion + + private void TraceRestUsage(IRestResponse response) + { + var total = GetHttpHeaderValue(response, "x-ratelimit-limit"); + var used = GetHttpHeaderValue(response, "x-ratelimit-used"); + var remaining = GetHttpHeaderValue(response, "x-ratelimit-remaining"); + + Log.Trace($"CoinApiDataQueueHandler.TraceRestUsage(): Used {used}, Remaining {remaining}, Total {total}"); + } + + private string GetHttpHeaderValue(IRestResponse response, string propertyName) + { + return response.Headers + .FirstOrDefault(x => x.Name == propertyName)? + .Value.ToString(); + } + + // WARNING: here to be called from tests to reduce explicitly the amount of request's output + protected void SetUpHistDataLimit(int limit) + { + HistoricalDataPerRequestLimit = limit; + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApiProduct.cs b/QuantConnect.CoinAPI/CoinApiProduct.cs new file mode 100644 index 0000000..1055342 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiProduct.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.CoinAPI +{ + /// + /// Coin API's available tariff plans (or products). + /// https://www.coinapi.io/Pricing + /// + public enum CoinApiProduct + { + /// + /// 100 daily requests, trades test only + /// + Free, + + /// + /// 1k daily requests, trades only + /// + Startup, + + /// + /// 10k daily requests, trades + quotes + /// + Streamer, + + /// + /// 100k daily requests, unlimited websocket + /// + Professional, + + /// + /// Contact Coin Api sales for more info + /// + Enterprise + } +} diff --git a/QuantConnect.CoinAPI/CoinApiSymbol.cs b/QuantConnect.CoinAPI/CoinApiSymbol.cs new file mode 100644 index 0000000..0f56083 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiSymbol.cs @@ -0,0 +1,42 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI +{ + public class CoinApiSymbol + { + [JsonProperty("symbol_id")] + public string SymbolId { get; set; } + + [JsonProperty("exchange_id")] + public string ExchangeId { get; set; } + + [JsonProperty("symbol_type")] + public string SymbolType { get; set; } + + [JsonProperty("asset_id_base")] + public string AssetIdBase { get; set; } + + [JsonProperty("asset_id_quote")] + public string AssetIdQuote { get; set; } + + public override string ToString() + { + return SymbolId; + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs new file mode 100644 index 0000000..01fdc88 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs @@ -0,0 +1,256 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using QuantConnect.Logging; +using QuantConnect.Securities; +using QuantConnect.Brokerages; +using QuantConnect.Configuration; + +namespace QuantConnect.CoinAPI +{ + /// + /// Provides the mapping between Lean symbols and CoinAPI symbols. + /// + /// For now we only support mapping for CoinbasePro (GDAX) and Bitfinex + public class CoinApiSymbolMapper : ISymbolMapper + { + private const string RestUrl = "https://rest.coinapi.io"; + private readonly string _apiKey = Config.Get("coinapi-api-key"); + private readonly bool _useLocalSymbolList = Config.GetBool("coinapi-use-local-symbol-list"); + + private readonly FileInfo _coinApiSymbolsListFile = new FileInfo( + Config.Get("coinapi-default-symbol-list-file", "CoinApiSymbols.json")); + // LEAN market <-> CoinAPI exchange id maps + public static readonly Dictionary MapMarketsToExchangeIds = new Dictionary + { + { Market.Coinbase, "COINBASE" }, + { Market.Bitfinex, "BITFINEX" }, + { Market.Binance, "BINANCE" }, + { Market.FTX, "FTX" }, + { Market.FTXUS, "FTXUS" }, + { Market.Kraken, "KRAKEN" }, + { Market.BinanceUS, "BINANCEUS" }, + { Market.Bybit, "BYBIT" }, + }; + private static readonly Dictionary MapExchangeIdsToMarkets = + MapMarketsToExchangeIds.ToDictionary(x => x.Value, x => x.Key); + + private static readonly Dictionary> CoinApiToLeanCurrencyMappings = + new Dictionary> + { + { + Market.Bitfinex, + new Dictionary + { + { "ABS", "ABYSS"}, + { "AIO", "AION"}, + { "ALG", "ALGO"}, + { "AMP", "AMPL"}, + { "ATO", "ATOM"}, + { "BCHABC", "BCH"}, + { "BCHSV", "BSV"}, + { "CSX", "CS"}, + { "CTX", "CTXC"}, + { "DOG", "MDOGE"}, + { "DRN", "DRGN"}, + { "DTX", "DT"}, + { "EDO", "PNT"}, + { "EUS", "EURS"}, + { "EUT", "EURT"}, + { "GSD", "GUSD"}, + { "HOPL", "HOT"}, + { "IOS", "IOST"}, + { "IOT", "IOTA"}, + { "LOO", "LOOM"}, + { "MIT", "MITH"}, + { "NCA", "NCASH"}, + { "OMN", "OMNI"}, + { "ORS", "ORST"}, + { "PAS", "PASS"}, + { "PKGO", "GOT"}, + { "POY", "POLY"}, + { "QSH", "QASH"}, + { "REP", "REP2"}, + { "SCR", "XD"}, + { "SNG", "SNGLS"}, + { "SPK", "SPANK"}, + { "STJ", "STORJ"}, + { "TSD", "TUSD"}, + { "UDC", "USDC"}, + { "ULTRA", "UOS"}, + { "USK", "USDK"}, + { "UTN", "UTNP"}, + { "VSY", "VSYS"}, + { "WBT", "WBTC"}, + { "XCH", "XCHF"}, + { "YGG", "YEED"} + } + } + }; + + // map LEAN symbols to CoinAPI symbol ids + private Dictionary _symbolMap = new Dictionary(); + + + /// + /// Creates a new instance of the class + /// + public CoinApiSymbolMapper() + { + MapExchangeIdsToMarkets["BINANCEFTS"] = Market.Binance; + MapExchangeIdsToMarkets["BINANCEFTSC"] = Market.Binance; + + MapExchangeIdsToMarkets["BYBITSPOT"] = Market.Bybit; + + LoadSymbolMap(MapExchangeIdsToMarkets.Keys.ToArray()); + } + + /// + /// Converts a Lean symbol instance to a CoinAPI symbol id + /// + /// A Lean symbol instance + /// The CoinAPI symbol id + public string GetBrokerageSymbol(Symbol symbol) + { + string symbolId; + if (!_symbolMap.TryGetValue(symbol, out symbolId)) + { + throw new Exception($"CoinApiSymbolMapper.GetBrokerageSymbol(): Symbol not found: {symbol}"); + } + + return symbolId; + } + + /// + /// Converts a CoinAPI symbol id to a Lean symbol instance + /// + /// The CoinAPI symbol id + /// The security type + /// The market + /// Expiration date of the security (if applicable) + /// The strike of the security (if applicable) + /// The option right of the security (if applicable) + /// A new Lean Symbol instance + public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, + DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = OptionRight.Call) + { + var parts = brokerageSymbol.Split('_'); + if (parts.Length != 4) + { + throw new Exception($"CoinApiSymbolMapper.GetLeanSymbol(): Unsupported SymbolId: {brokerageSymbol}"); + } + + string symbolMarket; + if (!MapExchangeIdsToMarkets.TryGetValue(parts[0], out symbolMarket)) + { + throw new Exception($"CoinApiSymbolMapper.GetLeanSymbol(): Unsupported ExchangeId: {parts[0]}"); + } + + var baseCurrency = ConvertCoinApiCurrencyToLeanCurrency(parts[2], symbolMarket); + var quoteCurrency = ConvertCoinApiCurrencyToLeanCurrency(parts[3], symbolMarket); + + var ticker = baseCurrency + quoteCurrency; + + return Symbol.Create(ticker, securityType, symbolMarket); + } + + /// + /// Returns the CoinAPI exchange id for the given market + /// + /// The Lean market + /// The CoinAPI exchange id + public string GetExchangeId(string market) + { + string exchangeId; + MapMarketsToExchangeIds.TryGetValue(market, out exchangeId); + + return exchangeId; + } + + private void LoadSymbolMap(string[] exchangeIds) + { + var list = string.Join(",", exchangeIds); + var json = string.Empty; + + if (_useLocalSymbolList) + { + if (!_coinApiSymbolsListFile.Exists) + { + throw new Exception($"CoinApiSymbolMapper.LoadSymbolMap(): File not found: {_coinApiSymbolsListFile.FullName}, please " + + $"download the latest symbol list from CoinApi."); + } + json = File.ReadAllText(_coinApiSymbolsListFile.FullName); + } + else + { + json = $"{RestUrl}/v1/symbols?filter_symbol_id={list}&apiKey={_apiKey}".DownloadData(); + } + + var result = JsonConvert.DeserializeObject>(json); + + // There were cases of entries in the CoinApiSymbols list with the following pattern: + // _SPOT___ + // Those cases should be ignored for SPOT prices. + foreach (var x in result + .Where(x => x.SymbolId.Split('_').Length == 4 && + // exclude Bitfinex BCH pre-2018-fork as for now we don't have historical mapping data + (x.ExchangeId != "BITFINEX" || x.AssetIdBase != "BCH" && x.AssetIdQuote != "BCH") + // solves the cases where we request 'binance' and get 'binanceus' + && MapExchangeIdsToMarkets.ContainsKey(x.ExchangeId))) + { + var market = MapExchangeIdsToMarkets[x.ExchangeId]; + + SecurityType securityType; + if (x.SymbolType == "SPOT") + { + securityType = SecurityType.Crypto; + } + else if (x.SymbolType == "PERPETUAL") + { + securityType = SecurityType.CryptoFuture; + } + else + { + continue; + } + var symbol = GetLeanSymbol(x.SymbolId, securityType, market); + + if (_symbolMap.ContainsKey(symbol)) + { + // skipping duplicate symbols. Kraken has both USDC/AD & USD/CAD symbols + Log.Error($"CoinApiSymbolMapper(): Duplicate symbol found {symbol} will be skipped!"); + continue; + } + _symbolMap[symbol] = x.SymbolId; + } + } + + private static string ConvertCoinApiCurrencyToLeanCurrency(string currency, string market) + { + Dictionary mappings; + if (CoinApiToLeanCurrencyMappings.TryGetValue(market, out mappings)) + { + string mappedCurrency; + if (mappings.TryGetValue(currency, out mappedCurrency)) + { + currency = mappedCurrency; + } + } + + return currency; + } + } +} diff --git a/QuantConnect.CoinAPI/Messages/BaseMessage.cs b/QuantConnect.CoinAPI/Messages/BaseMessage.cs new file mode 100644 index 0000000..a168252 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/BaseMessage.cs @@ -0,0 +1,25 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class BaseMessage + { + [JsonProperty("type")] + public string Type { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/ErrorMessage.cs b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs new file mode 100644 index 0000000..924c972 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs @@ -0,0 +1,25 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class ErrorMessage : BaseMessage + { + [JsonProperty("message")] + public string Message { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/HelloMessage.cs b/QuantConnect.CoinAPI/Messages/HelloMessage.cs new file mode 100644 index 0000000..cd79290 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/HelloMessage.cs @@ -0,0 +1,40 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class HelloMessage + { + [JsonProperty("type")] + public string Type { get; } = "hello"; + + [JsonProperty("apikey")] + public string ApiKey { get; set; } + + [JsonProperty("heartbeat")] + public bool Heartbeat { get; set; } + + [JsonProperty("subscribe_data_type")] + public string[] SubscribeDataType { get; set; } + + [JsonProperty("subscribe_filter_symbol_id")] + public string[] SubscribeFilterSymbolId { get; set; } + + [JsonProperty("subscribe_filter_asset_id")] + public string[] SubscribeFilterAssetId { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs new file mode 100644 index 0000000..ce5a83a --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class HistoricalDataMessage + { + [JsonProperty("time_period_start")] + public DateTime TimePeriodStart { get; set; } + + [JsonProperty("time_period_end")] + public DateTime TimePeriodEnd { get; set; } + + [JsonProperty("price_open")] + public decimal PriceOpen { get; set; } + + [JsonProperty("price_high")] + public decimal PriceHigh { get; set; } + + [JsonProperty("price_low")] + public decimal PriceLow { get; set; } + + [JsonProperty("price_close")] + public decimal PriceClose { get; set; } + + [JsonProperty("volume_traded")] + public decimal VolumeTraded { get; set; } + + [JsonProperty("trades_count")] + public int TradesCount { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/QuoteMessage.cs b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs new file mode 100644 index 0000000..59c273c --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class QuoteMessage : BaseMessage + { + [JsonProperty("symbol_id")] + public string SymbolId { get; set; } + + [JsonProperty("sequence")] + public long Sequence { get; set; } + + [JsonProperty("time_exchange")] + public DateTime TimeExchange { get; set; } + + [JsonProperty("time_coinapi")] + public DateTime TimeCoinApi { get; set; } + + [JsonProperty("ask_price")] + public decimal AskPrice { get; set; } + + [JsonProperty("ask_size")] + public decimal AskSize { get; set; } + + [JsonProperty("bid_price")] + public decimal BidPrice { get; set; } + + [JsonProperty("bid_size")] + public decimal BidSize { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/TradeMessage.cs b/QuantConnect.CoinAPI/Messages/TradeMessage.cs new file mode 100644 index 0000000..cfa6b31 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/TradeMessage.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class TradeMessage : BaseMessage + { + [JsonProperty("symbol_id")] + public string SymbolId { get; set; } + + [JsonProperty("sequence")] + public long Sequence { get; set; } + + [JsonProperty("time_exchange")] + public DateTime TimeExchange { get; set; } + + [JsonProperty("time_coinapi")] + public DateTime TimeCoinApi { get; set; } + + [JsonProperty("uuid")] + public string Uuid { get; set; } + + [JsonProperty("price")] + public decimal Price { get; set; } + + [JsonProperty("size")] + public decimal Size { get; set; } + + [JsonProperty("taker_side")] + public string TakerSide { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj b/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj index cdd3a6d..3096705 100644 --- a/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj +++ b/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj @@ -1,4 +1,4 @@ - + Release @@ -30,6 +30,7 @@ + From 4e8ebfbc6fc9ce222be72a0544cab0bfa27644c0 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 8 Feb 2024 02:18:15 +0200 Subject: [PATCH 04/34] feat: paste coinApi.Converter files from ToolBox --- .../CoinApiDataConverter.cs | 257 ++++++++++++++++++ .../CoinApiDataConverterProgram.cs | 38 +++ .../CoinApiDataReader.cs | 189 +++++++++++++ .../CoinApiEntryData.cs | 44 +++ .../QuantConnect.CoinAPI.Converter.csproj | 6 +- 5 files changed, 533 insertions(+), 1 deletion(-) create mode 100644 QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs create mode 100644 QuantConnect.CoinAPI.Converter/CoinApiDataConverterProgram.cs create mode 100644 QuantConnect.CoinAPI.Converter/CoinApiDataReader.cs create mode 100644 QuantConnect.CoinAPI.Converter/CoinApiEntryData.cs diff --git a/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs b/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs new file mode 100644 index 0000000..a8ece2a --- /dev/null +++ b/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs @@ -0,0 +1,257 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Data; +using QuantConnect.Util; +using System.Diagnostics; +using QuantConnect.Logging; +using QuantConnect.ToolBox; + +namespace QuantConnect.CoinAPI.Converter +{ + /// + /// Console application for converting CoinApi raw data into Lean data format for high resolutions (tick, second and minute) + /// + public class CoinApiDataConverter + { + /// + /// List of supported exchanges + /// + private static readonly HashSet SupportedMarkets = new[] + { + Market.Coinbase, + Market.Bitfinex, + Market.Binance, + Market.FTX, + Market.FTXUS, + Market.Kraken, + Market.BinanceUS, + Market.Bybit + }.ToHashSet(); + + private readonly DirectoryInfo _rawDataFolder; + private readonly DirectoryInfo _destinationFolder; + private readonly SecurityType _securityType; + private readonly DateTime _processingDate; + private readonly string _market; + + /// + /// CoinAPI data converter. + /// + /// the processing date. + /// path to the raw data folder. + /// destination of the newly generated files. + /// The security type to process + /// The market to process (optional). Defaults to processing all markets in parallel. + public CoinApiDataConverter(DateTime date, string rawDataFolder, string destinationFolder, string market = null, SecurityType securityType = SecurityType.Crypto) + { + _market = string.IsNullOrWhiteSpace(market) + ? null + : market.ToLowerInvariant(); + + _processingDate = date; + _securityType = securityType; + _rawDataFolder = new DirectoryInfo(Path.Combine(rawDataFolder, "crypto", "coinapi")); + if (!_rawDataFolder.Exists) + { + throw new ArgumentException($"CoinApiDataConverter(): Source folder not found: {_rawDataFolder.FullName}"); + } + + _destinationFolder = new DirectoryInfo(destinationFolder); + _destinationFolder.Create(); + } + + /// + /// Runs this instance. + /// + /// + public bool Run() + { + var stopwatch = Stopwatch.StartNew(); + + var symbolMapper = new CoinApiSymbolMapper(); + var success = true; + + // There were cases of files with with an extra suffix, following pattern: + // --_SPOT___.csv.gz + // Those cases should be ignored for SPOT prices. + var tradesFolder = new DirectoryInfo( + Path.Combine( + _rawDataFolder.FullName, + "trades", + _processingDate.ToStringInvariant(DateFormat.EightCharacter))); + + var quotesFolder = new DirectoryInfo( + Path.Combine( + _rawDataFolder.FullName, + "quotes", + _processingDate.ToStringInvariant(DateFormat.EightCharacter))); + + var rawMarket = _market != null && + CoinApiSymbolMapper.MapMarketsToExchangeIds.TryGetValue(_market, out var rawMarketValue) + ? rawMarketValue + : null; + + var securityTypeFilter = (string name) => name.Contains("_SPOT_"); + if (_securityType == SecurityType.CryptoFuture) + { + securityTypeFilter = (string name) => name.Contains("_FTS_") || name.Contains("_PERP_"); + } + + // Distinct by tick type and first two parts of the raw file name, separated by '-'. + // This prevents us from double processing the same ticker twice, in case we're given + // two raw data files for the same symbol. Related: https://github.com/QuantConnect/Lean/pull/3262 + var apiDataReader = new CoinApiDataReader(symbolMapper); + var filesToProcessCandidates = tradesFolder.EnumerateFiles("*.gz") + .Concat(quotesFolder.EnumerateFiles("*.gz")) + .Where(f => securityTypeFilter(f.Name) && (rawMarket == null || f.Name.Contains(rawMarket))) + .Where(f => f.Name.Split('_').Length == 4) + .ToList(); + + var filesToProcessKeys = new HashSet(); + var filesToProcess = new List(); + + foreach (var candidate in filesToProcessCandidates) + { + try + { + var entryData = apiDataReader.GetCoinApiEntryData(candidate, _processingDate, _securityType); + CurrencyPairUtil.DecomposeCurrencyPair(entryData.Symbol, out var baseCurrency, out var quoteCurrency); + + if (!candidate.FullName.Contains(baseCurrency) && !candidate.FullName.Contains(quoteCurrency)) + { + throw new Exception($"Skipping {candidate.FullName} we have the wrong symbol {entryData.Symbol}!"); + } + + var key = candidate.Directory.Parent.Name + entryData.Symbol.ID; + if (filesToProcessKeys.Add(key)) + { + // Separate list from HashSet to preserve ordering of viable candidates + filesToProcess.Add(candidate); + } + } + catch (Exception err) + { + // Most likely the exchange isn't supported. Log exception message to avoid excessive stack trace spamming in console output + Log.Error(err.Message); + } + } + + Parallel.ForEach(filesToProcess, (file, loopState) => + { + Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}..."); + try + { + ProcessEntry(apiDataReader, file); + } + catch (Exception e) + { + Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}"); + success = false; + loopState.Break(); + } + } + ); + + Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}"); + return success; + } + + /// + /// Processes the entry. + /// + /// The coinapi data reader. + /// The file. + private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file) + { + var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate, _securityType); + + if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market)) + { + // only convert data for supported exchanges + return; + } + + var tickData = coinapiDataReader.ProcessCoinApiEntry(entryData, file); + + // in some cases the first data points from '_processingDate' get's included in the previous date file + // so we will ready previous date data and drop most of it just to save these midnight ticks + var yesterdayDate = _processingDate.AddDays(-1); + var yesterdaysFile = new FileInfo(file.FullName.Replace( + _processingDate.ToStringInvariant(DateFormat.EightCharacter), + yesterdayDate.ToStringInvariant(DateFormat.EightCharacter))); + if (yesterdaysFile.Exists) + { + var yesterdaysEntryData = coinapiDataReader.GetCoinApiEntryData(yesterdaysFile, yesterdayDate, _securityType); + tickData = tickData.Concat(coinapiDataReader.ProcessCoinApiEntry(yesterdaysEntryData, yesterdaysFile)); + } + else + { + Log.Error($"CoinApiDataConverter(): yesterdays data file not found '{yesterdaysFile.FullName}'"); + } + + // materialize the enumerable into a list, since we need to enumerate over it twice + var ticks = tickData.Where(tick => tick.Time.Date == _processingDate) + .OrderBy(t => t.Time) + .ToList(); + + var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); + writer.Write(ticks); + + Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}"); + var consolidators = new List(); + + if (entryData.TickType == TickType.Trade) + { + consolidators.AddRange(new[] + { + new TradeTickAggregator(Resolution.Second), + new TradeTickAggregator(Resolution.Minute) + }); + } + else + { + consolidators.AddRange(new[] + { + new QuoteTickAggregator(Resolution.Second), + new QuoteTickAggregator(Resolution.Minute) + }); + } + + foreach (var tick in ticks) + { + if (tick.Suspicious) + { + // When CoinAPI loses connectivity to the exchange, they indicate + // it in the data by providing a value of `-1` for bid/ask price. + // We will keep it in tick data, but will remove it from consolidated data. + continue; + } + + foreach (var consolidator in consolidators) + { + consolidator.Update(tick); + } + } + + foreach (var consolidator in consolidators) + { + writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); + writer.Write(consolidator.Flush()); + } + } + } +} diff --git a/QuantConnect.CoinAPI.Converter/CoinApiDataConverterProgram.cs b/QuantConnect.CoinAPI.Converter/CoinApiDataConverterProgram.cs new file mode 100644 index 0000000..dca301d --- /dev/null +++ b/QuantConnect.CoinAPI.Converter/CoinApiDataConverterProgram.cs @@ -0,0 +1,38 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Globalization; + +namespace QuantConnect.CoinAPI.Converter +{ + /// + /// Coin API Main entry point for ToolBox. + /// + public static class CoinApiDataConverterProgram + { + public static void CoinApiDataProgram(string date, string rawDataFolder, string destinationFolder, string market, string securityType) + { + var processingDate = DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture); + var typeToProcess = SecurityType.Crypto; + if (!string.IsNullOrEmpty(securityType)) + { + typeToProcess = (SecurityType)Enum.Parse(typeof(SecurityType), securityType, true); + } + var converter = new CoinApiDataConverter(processingDate, rawDataFolder, destinationFolder, market, typeToProcess); + converter.Run(); + } + } +} diff --git a/QuantConnect.CoinAPI.Converter/CoinApiDataReader.cs b/QuantConnect.CoinAPI.Converter/CoinApiDataReader.cs new file mode 100644 index 0000000..6010c9c --- /dev/null +++ b/QuantConnect.CoinAPI.Converter/CoinApiDataReader.cs @@ -0,0 +1,189 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Globalization; +using QuantConnect.Logging; +using System.IO.Compression; +using QuantConnect.Brokerages; +using QuantConnect.Data.Market; +using CompressionMode = System.IO.Compression.CompressionMode; + +namespace QuantConnect.CoinAPI.Converter +{ + /// + /// Reader class for CoinAPI crypto raw data. + /// + public class CoinApiDataReader + { + private readonly ISymbolMapper _symbolMapper; + + /// + /// Creates a new instance of the class + /// + /// The symbol mapper + public CoinApiDataReader(ISymbolMapper symbolMapper) + { + _symbolMapper = symbolMapper; + } + + /// + /// Gets the coin API entry data. + /// + /// The source file. + /// The processing date. + /// The security type of this file. + /// + public CoinApiEntryData GetCoinApiEntryData(FileInfo file, DateTime processingDate, SecurityType securityType) + { + // crypto///-563-BITFINEX_SPOT_BTC_USD.csv.gz + var tickType = file.FullName.Contains("trades", StringComparison.InvariantCultureIgnoreCase) ? TickType.Trade : TickType.Quote; + + var symbolId = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(file.Name)).Split('-').Last(); + + var symbol = _symbolMapper.GetLeanSymbol(symbolId, securityType, null); + + return new CoinApiEntryData + { + Name = file.Name, + Symbol = symbol, + TickType = tickType, + Date = processingDate + }; + } + + /// + /// Gets an list of ticks for a given CoinAPI source file. + /// + /// The entry data. + /// The file. + /// + /// CoinApiDataReader.ProcessCoinApiEntry(): CSV header not found for entry name: {entryData.Name} + public IEnumerable ProcessCoinApiEntry(CoinApiEntryData entryData, FileInfo file) + { + Log.Trace("CoinApiDataReader.ProcessTarEntry(): Processing " + + $"{entryData.Symbol.ID.Market}-{entryData.Symbol.Value}-{entryData.TickType}-{entryData.Symbol.SecurityType} " + + $"for {entryData.Date:yyyy-MM-dd}"); + + + using (var stream = new GZipStream(file.OpenRead(), CompressionMode.Decompress)) + using (var reader = new StreamReader(stream)) + { + var headerLine = reader.ReadLine(); + if (headerLine == null) + { + throw new Exception($"CoinApiDataReader.ProcessCoinApiEntry(): CSV header not found for entry name: {entryData.Name}"); + } + + var headerParts = headerLine.Split(';').ToList(); + + var tickList = entryData.TickType == TickType.Trade + ? ParseTradeData(entryData.Symbol, reader, headerParts) + : ParseQuoteData(entryData.Symbol, reader, headerParts); + + foreach (var tick in tickList) + { + yield return tick; + } + } + } + + /// + /// Parses CoinAPI trade data. + /// + /// The symbol. + /// The reader. + /// The header parts. + /// + private IEnumerable ParseTradeData(Symbol symbol, StreamReader reader, List headerParts) + { + var columnTime = headerParts.FindIndex(x => x == "time_exchange"); + var columnPrice = headerParts.FindIndex(x => x == "price"); + var columnQuantity = headerParts.FindIndex(x => x == "base_amount"); + + string line; + while ((line = reader.ReadLine()) != null) + { + var lineParts = line.Split(';'); + + var time = DateTime.Parse(lineParts[columnTime], CultureInfo.InvariantCulture); + var price = lineParts[columnPrice].ToDecimal(); + var quantity = lineParts[columnQuantity].ToDecimal(); + + yield return new Tick + { + Symbol = symbol, + Time = time, + Value = price, + Quantity = quantity, + TickType = TickType.Trade, + Suspicious = price == -1 + }; + } + } + + /// + /// Parses CoinAPI quote data. + /// + /// The symbol. + /// The reader. + /// The header parts. + /// + private IEnumerable ParseQuoteData(Symbol symbol, StreamReader reader, List headerParts) + { + var columnTime = headerParts.FindIndex(x => x == "time_exchange"); + var columnAskPrice = headerParts.FindIndex(x => x == "ask_px"); + var columnAskSize = headerParts.FindIndex(x => x == "ask_sx"); + var columnBidPrice = headerParts.FindIndex(x => x == "bid_px"); + var columnBidSize = headerParts.FindIndex(x => x == "bid_sx"); + + var previousAskPrice = 0m; + var previousBidPrice = 0m; + + string line; + while ((line = reader.ReadLine()) != null) + { + var lineParts = line.Split(';'); + + var time = DateTime.Parse(lineParts[columnTime], CultureInfo.InvariantCulture); + var askPrice = lineParts[columnAskPrice].ToDecimal(); + var askSize = lineParts[columnAskSize].ToDecimal(); + var bidPrice = lineParts[columnBidPrice].ToDecimal(); + var bidSize = lineParts[columnBidSize].ToDecimal(); + + if (askPrice == previousAskPrice && bidPrice == previousBidPrice) + { + // only save quote if bid price or ask price changed + continue; + } + + previousAskPrice = askPrice; + previousBidPrice = bidPrice; + + yield return new Tick + { + Symbol = symbol, + Time = time, + AskPrice = askPrice, + AskSize = askSize, + BidPrice = bidPrice, + BidSize = bidSize, + TickType = TickType.Quote, + Suspicious = bidPrice == -1m || askPrice == -1m + }; + } + } + } +} diff --git a/QuantConnect.CoinAPI.Converter/CoinApiEntryData.cs b/QuantConnect.CoinAPI.Converter/CoinApiEntryData.cs new file mode 100644 index 0000000..d10847e --- /dev/null +++ b/QuantConnect.CoinAPI.Converter/CoinApiEntryData.cs @@ -0,0 +1,44 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.CoinAPI.Converter +{ + /// + /// Contains information extracted from CoinAPI entry name + /// + public class CoinApiEntryData + { + /// + /// The entry name + /// + public string Name { get; set; } + + /// + /// The LEAN symbol + /// + public Symbol Symbol { get; set; } + + /// + /// The tick type (Trade or Quote) + /// + public TickType TickType { get; set; } + + /// + /// The date of the entry + /// + public DateTime Date { get; set; } + } +} \ No newline at end of file diff --git a/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj b/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj index 8f25f33..4e30faf 100644 --- a/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj +++ b/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj @@ -1,4 +1,4 @@ - + Release @@ -28,6 +28,10 @@ bin\Release\ + + + + From 64b00f98d68d2623f11518d5538e8f39fe690b85 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 8 Feb 2024 02:19:56 +0200 Subject: [PATCH 05/34] feat: paste coinApi.test files from ToolBox feat: TestSetup initializer --- .../CoinAPIHistoryProviderTests.cs | 120 ++++++++++++++++++ .../CoinAPISymbolMapperTests.cs | 62 +++++++++ QuantConnect.CoinAPI.Tests/TestSetup.cs | 64 ++++++++++ 3 files changed, 246 insertions(+) create mode 100644 QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs create mode 100644 QuantConnect.CoinAPI.Tests/TestSetup.cs diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs new file mode 100644 index 0000000..3d9b78a --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -0,0 +1,120 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Data.Market; +using QuantConnect.Securities; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinAPIHistoryProviderTests + { + private static readonly Symbol _CoinbaseBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.GDAX); + private static readonly Symbol _BitfinexBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); + private readonly CoinApiDataQueueHandlerMock _coinApiDataQueueHandler = new CoinApiDataQueueHandlerMock(); + + // -- DATA TO TEST -- + private static TestCaseData[] TestData => new[] + { + // No data - invalid resolution or data type, or period is more than limit + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Tick, typeof(TradeBar), 100, false), + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Daily, typeof(QuoteBar), 100, false), + // Has data + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Minute, typeof(TradeBar), 216, true), + new TestCaseData(_CoinbaseBtcUsdSymbol, Resolution.Minute, typeof(TradeBar), 342, true), + new TestCaseData(_CoinbaseBtcUsdSymbol, Resolution.Hour, typeof(TradeBar), 107, true), + new TestCaseData(_CoinbaseBtcUsdSymbol, Resolution.Daily, typeof(TradeBar), 489, true), + // Can get data for resolution second + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Second, typeof(TradeBar), 300, true) + }; + + [Test] + [TestCaseSource(nameof(TestData))] + public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNonEmptyResult) + { + _coinApiDataQueueHandler.SetUpHistDataLimit(100); + + var nowUtc = DateTime.UtcNow; + var periodTimeSpan = TimeSpan.FromTicks(resolution.ToTimeSpan().Ticks * period); + var startTimeUtc = nowUtc.Add(-periodTimeSpan); + + var historyRequests = new[] + { + new HistoryRequest(startTimeUtc, nowUtc, dataType, symbol, resolution, + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), TimeZones.Utc, + resolution, true, false, DataNormalizationMode.Raw, TickType.Trade) + }; + + var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc).ToArray(); + + if (isNonEmptyResult) + { + // For resolution larger than second do more tests + if (resolution > Resolution.Second) + { + Assert.AreEqual(period, slices.Length); + + var firstSliceTradeBars = slices.First().Bars.Values; + + Assert.True(firstSliceTradeBars.Select(x => x.Symbol).Contains(symbol)); + + firstSliceTradeBars.DoForEach(tb => + { + var resTimeSpan = resolution.ToTimeSpan(); + Assert.AreEqual(resTimeSpan, tb.Period); + Assert.AreEqual(startTimeUtc.RoundUp(resTimeSpan), tb.Time); + }); + + var lastSliceTradeBars = slices.Last().Bars.Values; + + lastSliceTradeBars.DoForEach(tb => + { + var resTimeSpan = resolution.ToTimeSpan(); + Assert.AreEqual(resTimeSpan, tb.Period); + Assert.AreEqual(nowUtc.RoundDown(resTimeSpan), tb.Time); + }); + } + // For res. second data counts, start/end dates may slightly vary from historical request's + // Make sure just that resolution is correct and amount is positive numb. + else + { + Assert.IsTrue(slices.Length > 0); + Assert.AreEqual(resolution.ToTimeSpan(), slices.First().Bars.Values.FirstOrDefault()?.Period); + } + + // Slices are ordered by time + Assert.That(slices, Is.Ordered.By("Time")); + } + else + { + // Empty + Assert.IsEmpty(slices); + } + } + + public class CoinApiDataQueueHandlerMock : CoinApiDataQueueHandler + { + public new void SetUpHistDataLimit(int limit) + { + base.SetUpHistDataLimit(limit); + } + } + + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs new file mode 100644 index 0000000..f937e9e --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs @@ -0,0 +1,62 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinAPISymbolMapperTests + { + private CoinApiSymbolMapper _coinApiSymbolMapper; + + [OneTimeSetUp] + public void SetUp() + { + _coinApiSymbolMapper = new CoinApiSymbolMapper(); + } + + [TestCase("COINBASE_SPOT_BTC_USD", "BTCUSD", Market.Coinbase)] + [TestCase("COINBASE_SPOT_BCH_USD", "BCHUSD", Market.Coinbase)] + [TestCase("BITFINEX_SPOT_BTC_USD", "BTCUSD", Market.Bitfinex)] + [TestCase("BITFINEX_SPOT_BCHABC_USD", "BCHUSD", Market.Bitfinex)] + [TestCase("BITFINEX_SPOT_BCHSV_USD", "BSVUSD", Market.Bitfinex)] + [TestCase("BITFINEX_SPOT_ABS_USD", "ABYSSUSD", Market.Bitfinex)] + public void ReturnsCorrectLeanSymbol(string coinApiSymbolId, string leanTicker, string market) + { + var symbol = _coinApiSymbolMapper.GetLeanSymbol(coinApiSymbolId, SecurityType.Crypto, string.Empty); + + Assert.That(symbol.Value, Is.EqualTo(leanTicker)); + Assert.That(symbol.ID.SecurityType, Is.EqualTo(SecurityType.Crypto)); + Assert.That(symbol.ID.Market, Is.EqualTo(market)); + } + + [TestCase("BTCUSD", Market.Coinbase, "COINBASE_SPOT_BTC_USD")] + [TestCase("BCHUSD", Market.Coinbase, "COINBASE_SPOT_BCH_USD")] + [TestCase("BTCUSD", Market.Bitfinex, "BITFINEX_SPOT_BTC_USD")] + [TestCase("BCHUSD", Market.Bitfinex, "BITFINEX_SPOT_BCHABC_USD")] + [TestCase("BSVUSD", Market.Bitfinex, "BITFINEX_SPOT_BCHSV_USD")] + [TestCase("ABYSSUSD", Market.Bitfinex, "BITFINEX_SPOT_ABS_USD")] + public void ReturnsCorrectBrokerageSymbol(string leanTicker, string market, string coinApiSymbolId) + { + var symbol = Symbol.Create(leanTicker, SecurityType.Crypto, market); + + var symbolId = _coinApiSymbolMapper.GetBrokerageSymbol(symbol); + + Assert.That(symbolId, Is.EqualTo(coinApiSymbolId)); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/TestSetup.cs b/QuantConnect.CoinAPI.Tests/TestSetup.cs new file mode 100644 index 0000000..f881a82 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/TestSetup.cs @@ -0,0 +1,64 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using System.Collections; +using QuantConnect.Logging; +using QuantConnect.Configuration; + +namespace QuantConnect.CoinAPI.Tests +{ + [SetUpFixture] + public class TestSetup + { + [OneTimeSetUp] + public void GlobalSetup() + { + // Log.DebuggingEnabled = true; + Log.LogHandler = new CompositeLogHandler(); + Log.Trace("TestSetup(): starting..."); + ReloadConfiguration(); + } + + private static void ReloadConfiguration() + { + // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder + var dir = TestContext.CurrentContext.TestDirectory; + Environment.CurrentDirectory = dir; + Directory.SetCurrentDirectory(dir); + // reload config from current path + Config.Reset(); + + var environment = Environment.GetEnvironmentVariables(); + foreach (DictionaryEntry entry in environment) + { + var envKey = entry.Key.ToString(); + var value = entry.Value.ToString(); + + if (envKey.StartsWith("QC_")) + { + var key = envKey.Substring(3).Replace("_", "-").ToLower(); + + Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); + Config.Set(key, value); + } + } + + // resets the version among other things + Globals.Reset(); + } + } +} From 38212693d50fe28d031cf42d0166970bc474f242 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 9 Feb 2024 14:46:35 +0200 Subject: [PATCH 06/34] feat: divide HistoryProvider in file --- .../CoinApi.HistoryProvider.cs | 150 ++++++++++++++++++ .../CoinApiDataQueueHandler.cs | 134 +--------------- 2 files changed, 152 insertions(+), 132 deletions(-) create mode 100644 QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs new file mode 100644 index 0000000..849c4a4 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs @@ -0,0 +1,150 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NodaTime; +using RestSharp; +using Newtonsoft.Json; +using QuantConnect.Data; +using QuantConnect.Logging; +using QuantConnect.Data.Market; +using QuantConnect.CoinAPI.Messages; +using QuantConnect.Lean.Engine.DataFeeds; +using HistoryRequest = QuantConnect.Data.HistoryRequest; + +namespace QuantConnect.CoinAPI +{ + public partial class CoinApiDataQueueHandler + { + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + // NOP + } + + public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + var subscriptions = new List(); + foreach (var request in requests) + { + var history = GetHistory(request); + var subscription = CreateSubscription(request, history); + subscriptions.Add(subscription); + } + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + } + + public IEnumerable GetHistory(HistoryRequest historyRequest) + { + if (historyRequest.Symbol.SecurityType != SecurityType.Crypto && historyRequest.Symbol.SecurityType != SecurityType.CryptoFuture) + { + Log.Error($"CoinApiDataQueueHandler.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); + yield break; + } + + if (historyRequest.Resolution == Resolution.Tick) + { + Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical ticks, only OHLCV timeseries"); + yield break; + } + + if (historyRequest.DataType == typeof(QuoteBar)) + { + Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical QuoteBars , only TradeBars"); + yield break; + } + + var resolutionTimeSpan = historyRequest.Resolution.ToTimeSpan(); + var lastRequestedBarStartTime = historyRequest.EndTimeUtc.RoundDown(resolutionTimeSpan); + var currentStartTime = historyRequest.StartTimeUtc.RoundUp(resolutionTimeSpan); + var currentEndTime = lastRequestedBarStartTime; + + // Perform a check of the number of bars requested, this must not exceed a static limit + var dataRequestedCount = (currentEndTime - currentStartTime).Ticks + / resolutionTimeSpan.Ticks; + + if (dataRequestedCount > HistoricalDataPerRequestLimit) + { + currentEndTime = currentStartTime + + TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); + } + + while (currentStartTime < lastRequestedBarStartTime) + { + var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[historyRequest.Resolution]; + + // Time must be in ISO 8601 format + var coinApiStartTime = currentStartTime.ToStringInvariant("s"); + var coinApiEndTime = currentEndTime.ToStringInvariant("s"); + + // Construct URL for rest request + var baseUrl = + "https://rest.coinapi.io/v1/ohlcv/" + + $"{coinApiSymbol}/history?period_id={coinApiPeriod}&limit={HistoricalDataPerRequestLimit}" + + $"&time_start={coinApiStartTime}&time_end={coinApiEndTime}"; + + // Execute + var client = new RestClient(baseUrl); + var restRequest = new RestRequest(Method.GET); + restRequest.AddHeader("X-CoinAPI-Key", _apiKey); + var response = client.Execute(restRequest); + + // Log the information associated with the API Key's rest call limits. + TraceRestUsage(response); + + // Deserialize to array + var coinApiHistoryBars = JsonConvert.DeserializeObject(response.Content); + + // Can be no historical data for a short period interval + if (!coinApiHistoryBars.Any()) + { + Log.Error($"CoinApiDataQueueHandler.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]"); + continue; + } + + foreach (var ohlcv in coinApiHistoryBars) + { + yield return + new TradeBar(ohlcv.TimePeriodStart, historyRequest.Symbol, ohlcv.PriceOpen, ohlcv.PriceHigh, + ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, historyRequest.Resolution.ToTimeSpan()); + } + + currentStartTime = currentEndTime; + currentEndTime += TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); + } + } + + private void TraceRestUsage(IRestResponse response) + { + var total = GetHttpHeaderValue(response, "x-ratelimit-limit"); + var used = GetHttpHeaderValue(response, "x-ratelimit-used"); + var remaining = GetHttpHeaderValue(response, "x-ratelimit-remaining"); + + Log.Trace($"CoinApiDataQueueHandler.TraceRestUsage(): Used {used}, Remaining {remaining}, Total {total}"); + } + + private string GetHttpHeaderValue(IRestResponse response, string propertyName) + { + return response.Headers + .FirstOrDefault(x => x.Name == propertyName)? + .Value.ToString(); + } + + // WARNING: here to be called from tests to reduce explicitly the amount of request's output + protected void SetUpHistDataLimit(int limit) + { + HistoricalDataPerRequestLimit = limit; + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs index fe38c04..3a0865c 100644 --- a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -13,9 +13,6 @@ * limitations under the License. */ -using NodaTime; -using RestSharp; -using Newtonsoft.Json; using QuantConnect.Data; using QuantConnect.Util; using QuantConnect.Packets; @@ -25,18 +22,15 @@ using QuantConnect.Data.Market; using QuantConnect.Configuration; using System.Collections.Concurrent; -using QuantConnect.CoinAPI.Messages; using CoinAPI.WebSocket.V1.DataModels; -using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.HistoricalData; -using HistoryRequest = QuantConnect.Data.HistoryRequest; namespace QuantConnect.CoinAPI { /// /// An implementation of for CoinAPI /// - public class CoinApiDataQueueHandler : SynchronizingHistoryProvider, IDataQueueHandler + public partial class CoinApiDataQueueHandler : SynchronizingHistoryProvider, IDataQueueHandler { protected int HistoricalDataPerRequestLimit = 10000; private static readonly Dictionary _ResolutionToCoinApiPeriodMappings = new Dictionary @@ -82,7 +76,7 @@ public CoinApiDataQueueHandler() ? new[] { "trade" } : new[] { "trade", "quote" }; - Log.Trace($"CoinApiDataQueueHandler(): using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); + Log.Trace($"{nameof(CoinApiDataQueueHandler)}: using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); _client = new CoinApiWsClient(); _client.TradeEvent += OnTrade; @@ -379,129 +373,5 @@ private void OnError(object sender, Exception e) { Log.Error(e); } - - #region SynchronizingHistoryProvider - - public override void Initialize(HistoryProviderInitializeParameters parameters) - { - // NOP - } - - public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) - { - var subscriptions = new List(); - foreach (var request in requests) - { - var history = GetHistory(request); - var subscription = CreateSubscription(request, history); - subscriptions.Add(subscription); - } - return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); - } - - public IEnumerable GetHistory(HistoryRequest historyRequest) - { - if (historyRequest.Symbol.SecurityType != SecurityType.Crypto && historyRequest.Symbol.SecurityType != SecurityType.CryptoFuture) - { - Log.Error($"CoinApiDataQueueHandler.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); - yield break; - } - - if (historyRequest.Resolution == Resolution.Tick) - { - Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical ticks, only OHLCV timeseries"); - yield break; - } - - if (historyRequest.DataType == typeof(QuoteBar)) - { - Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical QuoteBars , only TradeBars"); - yield break; - } - - var resolutionTimeSpan = historyRequest.Resolution.ToTimeSpan(); - var lastRequestedBarStartTime = historyRequest.EndTimeUtc.RoundDown(resolutionTimeSpan); - var currentStartTime = historyRequest.StartTimeUtc.RoundUp(resolutionTimeSpan); - var currentEndTime = lastRequestedBarStartTime; - - // Perform a check of the number of bars requested, this must not exceed a static limit - var dataRequestedCount = (currentEndTime - currentStartTime).Ticks - / resolutionTimeSpan.Ticks; - - if (dataRequestedCount > HistoricalDataPerRequestLimit) - { - currentEndTime = currentStartTime - + TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); - } - - while (currentStartTime < lastRequestedBarStartTime) - { - var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); - var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[historyRequest.Resolution]; - - // Time must be in ISO 8601 format - var coinApiStartTime = currentStartTime.ToStringInvariant("s"); - var coinApiEndTime = currentEndTime.ToStringInvariant("s"); - - // Construct URL for rest request - var baseUrl = - "https://rest.coinapi.io/v1/ohlcv/" + - $"{coinApiSymbol}/history?period_id={coinApiPeriod}&limit={HistoricalDataPerRequestLimit}" + - $"&time_start={coinApiStartTime}&time_end={coinApiEndTime}"; - - // Execute - var client = new RestClient(baseUrl); - var restRequest = new RestRequest(Method.GET); - restRequest.AddHeader("X-CoinAPI-Key", _apiKey); - var response = client.Execute(restRequest); - - // Log the information associated with the API Key's rest call limits. - TraceRestUsage(response); - - // Deserialize to array - var coinApiHistoryBars = JsonConvert.DeserializeObject(response.Content); - - // Can be no historical data for a short period interval - if (!coinApiHistoryBars.Any()) - { - Log.Error($"CoinApiDataQueueHandler.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]"); - continue; - } - - foreach (var ohlcv in coinApiHistoryBars) - { - yield return - new TradeBar(ohlcv.TimePeriodStart, historyRequest.Symbol, ohlcv.PriceOpen, ohlcv.PriceHigh, - ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, historyRequest.Resolution.ToTimeSpan()); - } - - currentStartTime = currentEndTime; - currentEndTime += TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); - } - } - - #endregion - - private void TraceRestUsage(IRestResponse response) - { - var total = GetHttpHeaderValue(response, "x-ratelimit-limit"); - var used = GetHttpHeaderValue(response, "x-ratelimit-used"); - var remaining = GetHttpHeaderValue(response, "x-ratelimit-remaining"); - - Log.Trace($"CoinApiDataQueueHandler.TraceRestUsage(): Used {used}, Remaining {remaining}, Total {total}"); - } - - private string GetHttpHeaderValue(IRestResponse response, string propertyName) - { - return response.Headers - .FirstOrDefault(x => x.Name == propertyName)? - .Value.ToString(); - } - - // WARNING: here to be called from tests to reduce explicitly the amount of request's output - protected void SetUpHistDataLimit(int limit) - { - HistoricalDataPerRequestLimit = limit; - } } } From 8d8fe3ff56459f893db8f1661a8a1879ec2f2e19 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 9 Feb 2024 17:10:00 +0200 Subject: [PATCH 07/34] remove: does not support exchanges in SymbolMapper --- QuantConnect.CoinAPI/CoinApiSymbolMapper.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs index 01fdc88..949e77b 100644 --- a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs +++ b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs @@ -33,17 +33,17 @@ public class CoinApiSymbolMapper : ISymbolMapper private readonly FileInfo _coinApiSymbolsListFile = new FileInfo( Config.Get("coinapi-default-symbol-list-file", "CoinApiSymbols.json")); - // LEAN market <-> CoinAPI exchange id maps + /// + /// LEAN market <-> CoinAPI exchange id maps + /// + /// public static readonly Dictionary MapMarketsToExchangeIds = new Dictionary { { Market.Coinbase, "COINBASE" }, { Market.Bitfinex, "BITFINEX" }, { Market.Binance, "BINANCE" }, - { Market.FTX, "FTX" }, - { Market.FTXUS, "FTXUS" }, { Market.Kraken, "KRAKEN" }, { Market.BinanceUS, "BINANCEUS" }, - { Market.Bybit, "BYBIT" }, }; private static readonly Dictionary MapExchangeIdsToMarkets = MapMarketsToExchangeIds.ToDictionary(x => x.Value, x => x.Key); From fc09af347aff2409d336a1038a4338a762f0f111 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 9 Feb 2024 21:47:51 +0200 Subject: [PATCH 08/34] refactor: downgrade pckg Microsoft.NET.Test.Sdk --- QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj b/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj index a46f7dd..d73485b 100644 --- a/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj +++ b/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj @@ -19,7 +19,7 @@ - + all From dcb6b16e77fb251c2ae2a3e37955a25b915abf23 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 10 Feb 2024 02:44:57 +0200 Subject: [PATCH 09/34] feat: handle JsonSerializationException feat: simplify code feat: remove one warning msg --- .../CoinApiDataQueueHandler.cs | 2 +- QuantConnect.CoinAPI/CoinApiSymbolMapper.cs | 16 ++++++++-- .../Models/CoinApiErrorResponse.cs | 31 +++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs index 3a0865c..b1a1d2c 100644 --- a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -369,7 +369,7 @@ private Symbol GetSymbolUsingCache(string ticker) return result; } - private void OnError(object sender, Exception e) + private void OnError(object? sender, Exception e) { Log.Error(e); } diff --git a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs index 949e77b..8a3d94e 100644 --- a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs +++ b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs @@ -18,6 +18,7 @@ using QuantConnect.Securities; using QuantConnect.Brokerages; using QuantConnect.Configuration; +using QuantConnect.CoinAPI.Models; namespace QuantConnect.CoinAPI { @@ -125,8 +126,7 @@ public CoinApiSymbolMapper() /// The CoinAPI symbol id public string GetBrokerageSymbol(Symbol symbol) { - string symbolId; - if (!_symbolMap.TryGetValue(symbol, out symbolId)) + if (!_symbolMap.TryGetValue(symbol, out var symbolId)) { throw new Exception($"CoinApiSymbolMapper.GetBrokerageSymbol(): Symbol not found: {symbol}"); } @@ -199,7 +199,17 @@ private void LoadSymbolMap(string[] exchangeIds) json = $"{RestUrl}/v1/symbols?filter_symbol_id={list}&apiKey={_apiKey}".DownloadData(); } - var result = JsonConvert.DeserializeObject>(json); + List? result = new(); + + try + { + result = JsonConvert.DeserializeObject>(json); + } + catch (JsonSerializationException) + { + var error = JsonConvert.DeserializeObject(json); + throw new Exception(error.Error); + } // There were cases of entries in the CoinApiSymbols list with the following pattern: // _SPOT___ diff --git a/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs new file mode 100644 index 0000000..56133b1 --- /dev/null +++ b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs @@ -0,0 +1,31 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Models +{ + public readonly struct CoinApiErrorResponse + { + [JsonProperty("error")] + public string Error { get; } + + [JsonConstructor] + public CoinApiErrorResponse(string error) + { + Error = error; + } + } +} From a9826a202a617b1daa619070534e6405a6d4bd11 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 10 Feb 2024 02:46:54 +0200 Subject: [PATCH 10/34] feat: DQH tests with different param feat: GetBrokerage CryptoFuture Symbol Test feat: init with wrong api key test feat: helper class for tests --- .../CoinAPISymbolMapperTests.cs | 17 ++ .../CoinApiAdditionalTests.cs | 35 ++++ .../CoinApiDataQueueHandlerTest.cs | 192 ++++++++++++++++++ .../CoinApiTestHelper.cs | 128 ++++++++++++ 4 files changed, 372 insertions(+) create mode 100644 QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs diff --git a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs index f937e9e..7303fc1 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs @@ -58,5 +58,22 @@ public void ReturnsCorrectBrokerageSymbol(string leanTicker, string market, stri Assert.That(symbolId, Is.EqualTo(coinApiSymbolId)); } + + [TestCase("BTCUSDT", Market.Binance, "BINANCEFTS_PERP_BTC_USDT")] + public void ReturnsCorrectBrokerageFutureSymbol(string leanTicker, string market, string coinApiSymbolId) + { + var symbol = Symbol.Create(leanTicker, SecurityType.CryptoFuture, market); + + var symbolId = _coinApiSymbolMapper.GetBrokerageSymbol(symbol); + + Assert.That(symbolId, Is.EqualTo(coinApiSymbolId)); + } + + [TestCase("BTCUSDT", Market.Kraken)] + public void TryGetWrongBrokerageFutureSymbolThrowException(string leanTicker, string market) + { + var symbol = Symbol.Create(leanTicker, SecurityType.CryptoFuture, market); + Assert.Throws(() => _coinApiSymbolMapper.GetBrokerageSymbol(symbol)); + } } } diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs new file mode 100644 index 0000000..e483e32 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -0,0 +1,35 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Configuration; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinApiAdditionalTests + { + [Test] + public void ThrowsOnFailedAuthentication() + { + Config.Set("coinapi-api-key", "wrong-api-key"); + + Assert.Throws(() => + { + using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler(); + }); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs new file mode 100644 index 0000000..df58b9f --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -0,0 +1,192 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Data; +using QuantConnect.Logging; +using QuantConnect.Data.Market; +using System.Collections.Concurrent; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinApiDataQueueHandlerTest + { + private readonly CoinApiTestHelper _coinApiTestHelper = new(); + + private CoinApiDataQueueHandler _coinApiDataQueueHandler; + private CancellationTokenSource _cancellationTokenSource; + + [SetUp] + public void SetUp() + { + _coinApiDataQueueHandler = new(); + _cancellationTokenSource = new(); + } + + [TearDown] + public void TearDown() + { + if (_coinApiDataQueueHandler != null) + { + _coinApiDataQueueHandler.Dispose(); + } + _cancellationTokenSource.Dispose(); + } + + [Test] + public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() + { + var resetEvent = new AutoResetEvent(false); + var tradeBars = new List(); + var resolution = Resolution.Second; + var symbol = _coinApiTestHelper.BTCUSDCoinbase; + var dataConfig = _coinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); + + _coinApiTestHelper.ProcessFeed( + _coinApiDataQueueHandler.Subscribe(dataConfig, (s, e) => { }), + tick => + { + if (tick != null) + { + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + tradeBars.Add(tick); + + if (tradeBars.Count > 5) + { + resetEvent.Set(); + } + } + }, + () => _cancellationTokenSource.Cancel()); + + Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(20), _cancellationTokenSource.Token)); + + _coinApiDataQueueHandler.Unsubscribe(dataConfig); + + _coinApiTestHelper.AssertSymbol(tradeBars.First().Symbol, symbol); + + _coinApiTestHelper.AssertBaseData(tradeBars, resolution); + } + + [Test] + public void SubscribeToBTCUSDSecondOnDifferentMarkets() + { + var resetEvent = new AutoResetEvent(false); + var tradeBars = new List(); + var resolution = Resolution.Second; + var minimDataFromExchange = 5; + + var symbolBaseData = new ConcurrentDictionary> + { + [_coinApiTestHelper.BTCUSDKraken] = new(), + [_coinApiTestHelper.BTCUSDTBinance] = new(), + [_coinApiTestHelper.BTCUSDBitfinex] = new(), + [_coinApiTestHelper.BTCUSDCoinbase] = new() + }; + + var dataConfigs = new List(); + foreach (var symbol in symbolBaseData.Keys) + { + dataConfigs.Add(_coinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution)); + } + + foreach (var config in dataConfigs) + { + _coinApiTestHelper.ProcessFeed( + _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), + tick => + { + if (tick != null) + { + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + symbolBaseData[tick.Symbol].Add(tick); + } + }, + () => + { + _cancellationTokenSource.Cancel(); + }); + } + + resetEvent.WaitOne(TimeSpan.FromSeconds(30), _cancellationTokenSource.Token); + + foreach (var data in symbolBaseData) + { + if (data.Value.Count > minimDataFromExchange) + { + Log.Debug($"Unsubscribe: Symbol: {data.Key}, BaseData.Count: {data.Value.Count}"); + var config = dataConfigs.Where(x => x.Symbol == data.Key).First(); + _coinApiDataQueueHandler.Unsubscribe(config); + dataConfigs.Remove(config); + } + } + + if (dataConfigs.Count != 0) + { + resetEvent.WaitOne(TimeSpan.FromSeconds(20), _cancellationTokenSource.Token); + } + + foreach (var config in dataConfigs) + { + _coinApiDataQueueHandler.Unsubscribe(config); + } + + foreach (var data in symbolBaseData.Values) + { + _coinApiTestHelper.AssertBaseData(data, resolution); + } + } + + [Test] + public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() + { + var resetEvent = new AutoResetEvent(false); + var resolution = Resolution.Tick; + var tickData = new List(); + var symbol = _coinApiTestHelper.BTCUSDTFutureBinance; + var config = _coinApiTestHelper.GetSubscriptionTickDataConfigs(symbol); + + _coinApiTestHelper.ProcessFeed( + _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), + tick => + { + if (tick != null) + { + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + tickData.Add(tick); + + if (tickData.Count > 5) + { + resetEvent.Set(); + } + } + }, + () => + { + _cancellationTokenSource.Cancel(); + }); + + resetEvent.WaitOne(TimeSpan.FromSeconds(30), _cancellationTokenSource.Token); + + _coinApiDataQueueHandler.Unsubscribe(config); + + _coinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); + + _coinApiTestHelper.AssertBaseData(tickData, resolution); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs new file mode 100644 index 0000000..70586f2 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -0,0 +1,128 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Data; +using QuantConnect.Logging; +using QuantConnect.Data.Market; + +namespace QuantConnect.CoinAPI.Tests +{ + public class CoinApiTestHelper + { + public readonly Symbol BTCUSDKraken = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Kraken); + public readonly Symbol BTCUSDBitfinex = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); + public readonly Symbol BTCUSDCoinbase = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); + public readonly Symbol BTCUSDTBinance = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.Binance); + public readonly Symbol BTCUSDTBinanceUS = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.BinanceUS); + + /// + /// PERPETUAL BTCUSDT + /// + public readonly Symbol BTCUSDTFutureBinance = Symbol.Create("BTCUSDT", SecurityType.CryptoFuture, Market.Binance); + + public void AssertSymbol(Symbol actualSymbol, Symbol expectedSymbol) + { + Assert.IsTrue(actualSymbol == expectedSymbol, $"Unexpected Symbol: Expected {expectedSymbol}, but received {actualSymbol}."); + } + + public void AssertBaseData(List tradeBars, Resolution expectedResolution) + { + Assert.Greater(tradeBars.Count, 0); + foreach (var tick in tradeBars) + { + Assert.IsNotNull(tick); + Assert.Greater(tick.Price, 0); + Assert.Greater(tick.Value, 0); + Assert.Greater(tick.Time, DateTime.UnixEpoch); + Assert.Greater(tick.EndTime, DateTime.UnixEpoch); + + if (tick.DataType == MarketDataType.Tick) + { + return; + } + + Assert.IsTrue(tick.DataType == MarketDataType.TradeBar || tick.DataType == MarketDataType.QuoteBar, $"Unexpected data type: Expected TradeBar or QuoteBar, but received {tick.DataType}."); + + switch (tick) + { + case TradeBar trade: + Assert.Greater(trade.Low, 0); + Assert.Greater(trade.Open, 0); + Assert.Greater(trade.High, 0); + Assert.Greater(trade.Close, 0); + Assert.Greater(trade.Volume, 0); + Assert.IsTrue(trade.Period.ToHigherResolutionEquivalent(true) == expectedResolution); + break; + default: + Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(AssertBaseData)}: The tick type doesn't support"); + break; + } + } + } + + public void ProcessFeed(IEnumerator enumerator, Action? callback = null, Action? throwExceptionCallback = null) + { + Task.Factory.StartNew(() => + { + try + { + while (enumerator.MoveNext()) + { + BaseData tick = enumerator.Current; + callback?.Invoke(tick); + + Thread.Sleep(1000); + } + } + catch + { + throw; + } + }).ContinueWith(task => + { + if (throwExceptionCallback != null) + { + throwExceptionCallback(); + } + Log.Error("The throwExceptionCallback is null."); + }, TaskContinuationOptions.OnlyOnFaulted); + } + + public SubscriptionDataConfig GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) + { + return GetSubscriptionDataConfig(symbol, resolution); + } + + public SubscriptionDataConfig GetSubscriptionTickDataConfigs(Symbol symbol) + { + return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Trade); + } + + private SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) + { + return new SubscriptionDataConfig( + typeof(T), + symbol, + resolution, + TimeZones.Utc, + TimeZones.Utc, + true, + extendedHours: false, + false); + } + } +} From b2b59b21632298cc4264230e2685e832cd06d4a8 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 12 Feb 2024 14:06:13 +0200 Subject: [PATCH 11/34] feat: forceTypeNameOnExisting set up to false for GetExportedValueByTypeName --- QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs index b1a1d2c..e890c30 100644 --- a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -69,7 +69,7 @@ public CoinApiDataQueueHandler() if (_dataAggregator == null) { _dataAggregator = - Composer.Instance.GetExportedValueByTypeName(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")); + Composer.Instance.GetExportedValueByTypeName(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); } var product = Config.GetValue("coinapi-product"); _streamingDataType = product < CoinApiProduct.Streamer From 6690715034bce77ad6b93466c62bbc40d744ab6b Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 12 Feb 2024 16:26:39 +0200 Subject: [PATCH 12/34] feat: add sync bash script in Converter project --- .gitignore | 1 + QuantConnect.CoinAPI.Converter/Program.cs | 46 ++++++++++++++++++- .../QuantConnect.CoinAPI.Converter.csproj | 6 +++ QuantConnect.CoinAPI.Converter/sync.sh | 40 ++++++++++++++++ 4 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 QuantConnect.CoinAPI.Converter/sync.sh diff --git a/.gitignore b/.gitignore index 866dd07..39e099c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ *.hex # QC Cloud Setup Bash Files +!QuantConnect.CoinAPI.Converter/*.sh *.sh # Include docker launch scripts for Mac/Linux !run_docker.sh diff --git a/QuantConnect.CoinAPI.Converter/Program.cs b/QuantConnect.CoinAPI.Converter/Program.cs index 3751555..ed2ac48 100644 --- a/QuantConnect.CoinAPI.Converter/Program.cs +++ b/QuantConnect.CoinAPI.Converter/Program.cs @@ -1,2 +1,44 @@ -// See https://aka.ms/new-console-template for more information -Console.WriteLine("Hello, World!"); +using QuantConnect.Logging; +using System.Globalization; +using QuantConnect.Configuration; + +namespace QuantConnect.CoinAPI.Converter +{ + internal class Program + { + public static string DataFleetDeploymentDate = "QC_DATAFLEET_DEPLOYMENT_DATE"; + + private static void Main(string[] args) + { + var rawDataFolder = new DirectoryInfo(Config.Get("raw-data-folder", "/raw")); + var temporaryOutputDirectory = Config.Get("temp-output-directory", "/temp-output-directory"); + // Allow for caller to specify which market they want to process + var market = args.Length != 0 && !string.IsNullOrWhiteSpace(args[0]) + ? args[0] + : null; + + var securityType = SecurityType.Crypto; + if (args.Length > 1 && !string.IsNullOrWhiteSpace(args[1])) + { + securityType = (SecurityType)Enum.Parse(typeof(SecurityType), args[1], true); + } + + Environment.SetEnvironmentVariable(DataFleetDeploymentDate, "20240211"); + + var processingDateValue = Environment.GetEnvironmentVariable(DataFleetDeploymentDate); + var processingDate = DateTime.ParseExact(processingDateValue, "yyyyMMdd", CultureInfo.InvariantCulture); + + Log.Trace($"Price.Crypto.CoinApi.Main(): Processing {processingDate} for market: {market ?? "*"} SecurityType: {securityType}"); + + var converter = new CoinApiDataConverter(processingDate, rawDataFolder.FullName, temporaryOutputDirectory, market, securityType); + + if (!converter.Run()) + { + Log.Error($"Price.Crypto.CoinApi.Main(): Processing CoinAPI for date {processingDate:yyyy-MM-dd} failed"); + Environment.Exit(1); + } + + Environment.Exit(0); + } + } +} \ No newline at end of file diff --git a/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj b/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj index 4e30faf..6f034cd 100644 --- a/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj +++ b/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj @@ -36,4 +36,10 @@ + + + PreserveNewest + + + diff --git a/QuantConnect.CoinAPI.Converter/sync.sh b/QuantConnect.CoinAPI.Converter/sync.sh new file mode 100644 index 0000000..e164442 --- /dev/null +++ b/QuantConnect.CoinAPI.Converter/sync.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +processing_date="${QC_DATAFLEET_DEPLOYMENT_DATE}" +processing_date_yesterday="$(date -d "${processing_date} -1 days" +%Y%m%d)" +market="*" +include="*" +filter="_SPOT*" + +if [ -n "$2" ]; then + filter="${2}*" +fi + +if [ -n "$1" ]; then + market=$(echo "${1}" | tr [a-z] [A-Z]) + include="*${market}${filter}" + echo "Downloading crypto TAQ for market: ${market} include ${include}" +else + echo "Downloading crypto TAQ for all markets" +fi + +# We store AWS creds and set the CoinAPI creds instead +aws_access_key_id="${AWS_ACCESS_KEY_ID}" +aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}" + +export AWS_ACCESS_KEY_ID="${COINAPI_AWS_ACCESS_KEY_ID}" +export AWS_SECRET_ACCESS_KEY="${COINAPI_AWS_SECRET_ACCESS_KEY}" + +# multipart corrupts files +aws configure set default.s3.multipart_threshold 100GB + +# We sync yesterdays file too because midnight data of 'processing_date' might be in the end of yesterdays file +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/trades/$processing_date_yesterday/$market/ /raw/crypto/coinapi/trades/$processing_date_yesterday/ --exclude='*' --include="${include}" +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/quotes/$processing_date_yesterday/$market/ /raw/crypto/coinapi/quotes/$processing_date_yesterday/ --exclude='*' --include="${include}" + +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/trades/$processing_date/$market/ /raw/crypto/coinapi/trades/$processing_date/ --exclude='*' --include="${include}" +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/quotes/$processing_date/$market/ /raw/crypto/coinapi/quotes/$processing_date/ --exclude='*' --include="${include}" + +# Restore AWS creds +export AWS_ACCESS_KEY_ID="$aws_access_key_id" +export AWS_SECRET_ACCESS_KEY="$aws_secret_access_key" \ No newline at end of file From f48c3ead1bfdc7009c0f7f569be06b5f4934c783 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 12 Feb 2024 17:17:04 +0200 Subject: [PATCH 13/34] remove: unsupported markets from Converter --- QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs b/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs index a8ece2a..c10286f 100644 --- a/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs +++ b/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs @@ -35,11 +35,8 @@ public class CoinApiDataConverter Market.Coinbase, Market.Bitfinex, Market.Binance, - Market.FTX, - Market.FTXUS, Market.Kraken, - Market.BinanceUS, - Market.Bybit + Market.BinanceUS }.ToHashSet(); private readonly DirectoryInfo _rawDataFolder; From ef973c6613cc08cdc959e0d708a8f0669a1e1e5d Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 12 Feb 2024 17:23:46 +0200 Subject: [PATCH 14/34] feat: ValidateSubscription --- .../CoinApiDataQueueHandler.cs | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs index e890c30..f9193f6 100644 --- a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -13,14 +13,22 @@ * limitations under the License. */ +using RestSharp; +using System.Net; +using System.Text; +using Newtonsoft.Json; +using QuantConnect.Api; using QuantConnect.Data; using QuantConnect.Util; using QuantConnect.Packets; using QuantConnect.Logging; +using Newtonsoft.Json.Linq; using CoinAPI.WebSocket.V1; using QuantConnect.Interfaces; using QuantConnect.Data.Market; using QuantConnect.Configuration; +using System.Security.Cryptography; +using System.Net.NetworkInformation; using System.Collections.Concurrent; using CoinAPI.WebSocket.V1.DataModels; using QuantConnect.Lean.Engine.HistoricalData; @@ -78,6 +86,8 @@ public CoinApiDataQueueHandler() Log.Trace($"{nameof(CoinApiDataQueueHandler)}: using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); + ValidateSubscription(); + _client = new CoinApiWsClient(); _client.TradeEvent += OnTrade; _client.QuoteEvent += OnQuote; @@ -373,5 +383,144 @@ private void OnError(object? sender, Exception e) { Log.Error(e); } + + private class ModulesReadLicenseRead : Api.RestResponse + { + [JsonProperty(PropertyName = "license")] + public string License; + + [JsonProperty(PropertyName = "organizationId")] + public string OrganizationId; + } + + /// + /// Validate the user of this project has permission to be using it via our web API. + /// + private static void ValidateSubscription() + { + try + { + const int productId = 333; + var userId = Config.GetInt("job-user-id"); + var token = Config.Get("api-access-token"); + var organizationId = Config.Get("job-organization-id", null); + // Verify we can authenticate with this user and token + var api = new ApiConnection(userId, token); + if (!api.Connected) + { + throw new ArgumentException("Invalid api user id or token, cannot authenticate subscription."); + } + // Compile the information we want to send when validating + var information = new Dictionary() + { + {"productId", productId}, + {"machineName", Environment.MachineName}, + {"userName", Environment.UserName}, + {"domainName", Environment.UserDomainName}, + {"os", Environment.OSVersion} + }; + // IP and Mac Address Information + try + { + var interfaceDictionary = new List>(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up)) + { + var interfaceInformation = new Dictionary(); + // Get UnicastAddresses + var addresses = nic.GetIPProperties().UnicastAddresses + .Select(uniAddress => uniAddress.Address) + .Where(address => !IPAddress.IsLoopback(address)).Select(x => x.ToString()); + // If this interface has non-loopback addresses, we will include it + if (!addresses.IsNullOrEmpty()) + { + interfaceInformation.Add("unicastAddresses", addresses); + // Get MAC address + interfaceInformation.Add("MAC", nic.GetPhysicalAddress().ToString()); + // Add Interface name + interfaceInformation.Add("name", nic.Name); + // Add these to our dictionary + interfaceDictionary.Add(interfaceInformation); + } + } + information.Add("networkInterfaces", interfaceDictionary); + } + catch (Exception) + { + // NOP, not necessary to crash if fails to extract and add this information + } + // Include our OrganizationId if specified + if (!string.IsNullOrEmpty(organizationId)) + { + information.Add("organizationId", organizationId); + } + var request = new RestRequest("modules/license/read", Method.POST) { RequestFormat = DataFormat.Json }; + request.AddParameter("application/json", JsonConvert.SerializeObject(information), ParameterType.RequestBody); + api.TryRequest(request, out ModulesReadLicenseRead result); + if (!result.Success) + { + throw new InvalidOperationException($"Request for subscriptions from web failed, Response Errors : {string.Join(',', result.Errors)}"); + } + + var encryptedData = result.License; + // Decrypt the data we received + DateTime? expirationDate = null; + long? stamp = null; + bool? isValid = null; + if (encryptedData != null) + { + // Fetch the org id from the response if it was not set, we need it to generate our validation key + if (string.IsNullOrEmpty(organizationId)) + { + organizationId = result.OrganizationId; + } + // Create our combination key + var password = $"{token}-{organizationId}"; + var key = SHA256.HashData(Encoding.UTF8.GetBytes(password)); + // Split the data + var info = encryptedData.Split("::"); + var buffer = Convert.FromBase64String(info[0]); + var iv = Convert.FromBase64String(info[1]); + // Decrypt our information + using var aes = new AesManaged(); + var decryptor = aes.CreateDecryptor(key, iv); + using var memoryStream = new MemoryStream(buffer); + using var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read); + using var streamReader = new StreamReader(cryptoStream); + var decryptedData = streamReader.ReadToEnd(); + if (!decryptedData.IsNullOrEmpty()) + { + var jsonInfo = JsonConvert.DeserializeObject(decryptedData); + expirationDate = jsonInfo["expiration"]?.Value(); + isValid = jsonInfo["isValid"]?.Value(); + stamp = jsonInfo["stamped"]?.Value(); + } + } + // Validate our conditions + if (!expirationDate.HasValue || !isValid.HasValue || !stamp.HasValue) + { + throw new InvalidOperationException("Failed to validate subscription."); + } + + var nowUtc = DateTime.UtcNow; + var timeSpan = nowUtc - Time.UnixTimeStampToDateTime(stamp.Value); + if (timeSpan > TimeSpan.FromHours(12)) + { + throw new InvalidOperationException("Invalid API response."); + } + if (!isValid.Value) + { + throw new ArgumentException($"Your subscription is not valid, please check your product subscriptions on our website."); + } + if (expirationDate < nowUtc) + { + throw new ArgumentException($"Your subscription expired {expirationDate}, please renew in order to use this product."); + } + } + catch (Exception e) + { + Log.Error($"{nameof(CoinApiDataQueueHandler)}.{nameof(ValidateSubscription)}: Failed during validation, shutting down. Error : {e.Message}"); + throw; + } + } } } From 33b67071450c6e0238f020d3d6c075b16d0f50dd Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 12 Feb 2024 19:48:29 +0200 Subject: [PATCH 15/34] fix: productId in ValidateSubscription --- QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs index f9193f6..a2e612b 100644 --- a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -400,7 +400,7 @@ private static void ValidateSubscription() { try { - const int productId = 333; + const int productId = 335; var userId = Config.GetInt("job-user-id"); var token = Config.Get("api-access-token"); var organizationId = Config.Get("job-organization-id", null); From 2c2f68031d88c0c4c775594f2d7f66b57726dac5 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 12 Feb 2024 22:15:16 +0200 Subject: [PATCH 16/34] refactor: make static of TestHelper class fix: deprecated GDAX to Coinbase Market in Symbol test --- .../CoinAPIHistoryProviderTests.cs | 2 +- .../CoinApiDataQueueHandlerTest.cs | 40 +++++++++---------- .../CoinApiTestHelper.cs | 26 ++++++------ 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index 3d9b78a..a30d625 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -25,7 +25,7 @@ namespace QuantConnect.CoinAPI.Tests [TestFixture] public class CoinAPIHistoryProviderTests { - private static readonly Symbol _CoinbaseBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.GDAX); + private static readonly Symbol _CoinbaseBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); private static readonly Symbol _BitfinexBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); private readonly CoinApiDataQueueHandlerMock _coinApiDataQueueHandler = new CoinApiDataQueueHandlerMock(); diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index df58b9f..f2a414e 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -25,8 +25,6 @@ namespace QuantConnect.CoinAPI.Tests [TestFixture] public class CoinApiDataQueueHandlerTest { - private readonly CoinApiTestHelper _coinApiTestHelper = new(); - private CoinApiDataQueueHandler _coinApiDataQueueHandler; private CancellationTokenSource _cancellationTokenSource; @@ -53,10 +51,10 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() var resetEvent = new AutoResetEvent(false); var tradeBars = new List(); var resolution = Resolution.Second; - var symbol = _coinApiTestHelper.BTCUSDCoinbase; - var dataConfig = _coinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); + var symbol = CoinApiTestHelper.BTCUSDCoinbase; + var dataConfig = CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); - _coinApiTestHelper.ProcessFeed( + CoinApiTestHelper.ProcessFeed( _coinApiDataQueueHandler.Subscribe(dataConfig, (s, e) => { }), tick => { @@ -77,9 +75,9 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() _coinApiDataQueueHandler.Unsubscribe(dataConfig); - _coinApiTestHelper.AssertSymbol(tradeBars.First().Symbol, symbol); + CoinApiTestHelper.AssertSymbol(tradeBars.First().Symbol, symbol); - _coinApiTestHelper.AssertBaseData(tradeBars, resolution); + CoinApiTestHelper.AssertBaseData(tradeBars, resolution); } [Test] @@ -92,21 +90,21 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() var symbolBaseData = new ConcurrentDictionary> { - [_coinApiTestHelper.BTCUSDKraken] = new(), - [_coinApiTestHelper.BTCUSDTBinance] = new(), - [_coinApiTestHelper.BTCUSDBitfinex] = new(), - [_coinApiTestHelper.BTCUSDCoinbase] = new() + [CoinApiTestHelper.BTCUSDKraken] = new(), + [CoinApiTestHelper.BTCUSDTBinance] = new(), + [CoinApiTestHelper.BTCUSDBitfinex] = new(), + [CoinApiTestHelper.BTCUSDCoinbase] = new() }; var dataConfigs = new List(); foreach (var symbol in symbolBaseData.Keys) { - dataConfigs.Add(_coinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution)); + dataConfigs.Add(CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution)); } foreach (var config in dataConfigs) { - _coinApiTestHelper.ProcessFeed( + CoinApiTestHelper.ProcessFeed( _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), tick => { @@ -147,7 +145,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() foreach (var data in symbolBaseData.Values) { - _coinApiTestHelper.AssertBaseData(data, resolution); + CoinApiTestHelper.AssertBaseData(data, resolution); } } @@ -157,10 +155,10 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() var resetEvent = new AutoResetEvent(false); var resolution = Resolution.Tick; var tickData = new List(); - var symbol = _coinApiTestHelper.BTCUSDTFutureBinance; - var config = _coinApiTestHelper.GetSubscriptionTickDataConfigs(symbol); + var symbol = CoinApiTestHelper.BTCUSDTFutureBinance; + var config = CoinApiTestHelper.GetSubscriptionTickDataConfigs(symbol); - _coinApiTestHelper.ProcessFeed( + CoinApiTestHelper.ProcessFeed( _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), tick => { @@ -181,12 +179,12 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() }); resetEvent.WaitOne(TimeSpan.FromSeconds(30), _cancellationTokenSource.Token); - + _coinApiDataQueueHandler.Unsubscribe(config); - _coinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); - - _coinApiTestHelper.AssertBaseData(tickData, resolution); + CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); + + CoinApiTestHelper.AssertBaseData(tickData, resolution); } } } diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs index 70586f2..29a88b9 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -21,25 +21,25 @@ namespace QuantConnect.CoinAPI.Tests { - public class CoinApiTestHelper + public static class CoinApiTestHelper { - public readonly Symbol BTCUSDKraken = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Kraken); - public readonly Symbol BTCUSDBitfinex = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); - public readonly Symbol BTCUSDCoinbase = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); - public readonly Symbol BTCUSDTBinance = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.Binance); - public readonly Symbol BTCUSDTBinanceUS = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.BinanceUS); + public static readonly Symbol BTCUSDKraken = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Kraken); + public static readonly Symbol BTCUSDBitfinex = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); + public static readonly Symbol BTCUSDCoinbase = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); + public static readonly Symbol BTCUSDTBinance = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.Binance); + public static readonly Symbol BTCUSDTBinanceUS = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.BinanceUS); /// /// PERPETUAL BTCUSDT /// - public readonly Symbol BTCUSDTFutureBinance = Symbol.Create("BTCUSDT", SecurityType.CryptoFuture, Market.Binance); + public static readonly Symbol BTCUSDTFutureBinance = Symbol.Create("BTCUSDT", SecurityType.CryptoFuture, Market.Binance); - public void AssertSymbol(Symbol actualSymbol, Symbol expectedSymbol) + public static void AssertSymbol(Symbol actualSymbol, Symbol expectedSymbol) { Assert.IsTrue(actualSymbol == expectedSymbol, $"Unexpected Symbol: Expected {expectedSymbol}, but received {actualSymbol}."); } - public void AssertBaseData(List tradeBars, Resolution expectedResolution) + public static void AssertBaseData(List tradeBars, Resolution expectedResolution) { Assert.Greater(tradeBars.Count, 0); foreach (var tick in tradeBars) @@ -74,7 +74,7 @@ public void AssertBaseData(List tradeBars, Resolution expectedResoluti } } - public void ProcessFeed(IEnumerator enumerator, Action? callback = null, Action? throwExceptionCallback = null) + public static void ProcessFeed(IEnumerator enumerator, Action? callback = null, Action? throwExceptionCallback = null) { Task.Factory.StartNew(() => { @@ -102,17 +102,17 @@ public void ProcessFeed(IEnumerator enumerator, Action? call }, TaskContinuationOptions.OnlyOnFaulted); } - public SubscriptionDataConfig GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) + public static SubscriptionDataConfig GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) { return GetSubscriptionDataConfig(symbol, resolution); } - public SubscriptionDataConfig GetSubscriptionTickDataConfigs(Symbol symbol) + public static SubscriptionDataConfig GetSubscriptionTickDataConfigs(Symbol symbol) { return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Trade); } - private SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) + private static SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) { return new SubscriptionDataConfig( typeof(T), From ca84e2e0fa69a98df3d08d39d27bd2a36bf37e28 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 12 Feb 2024 23:16:42 +0200 Subject: [PATCH 17/34] feat: CoinAPIDataDownloader fea: test of CoinAPIDataDownloader fix: reset config in wrong api key test --- .../CoinAPIDataDownloaderTests.cs | 110 ++++++++++++++++++ .../CoinApiAdditionalTests.cs | 2 + QuantConnect.CoinAPI/CoinAPIDataDownloader.cs | 78 +++++++++++++ .../CoinApi.HistoryProvider.cs | 13 ++- 4 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs create mode 100644 QuantConnect.CoinAPI/CoinAPIDataDownloader.cs diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs new file mode 100644 index 0000000..669d48a --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -0,0 +1,110 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Util; +using QuantConnect.Logging; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinAPIDataDownloaderTests + { + private CoinAPIDataDownloader _downloader; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _downloader = new(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + _downloader.DisposeSafely(); + } + + private static IEnumerable HistoricalValidDataTestCases + { + get + { + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Minute, new DateTime(2024, 1, 1, 20, 0, 0), new DateTime(2024, 1, 1, 21, 0, 0)); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDKraken, Resolution.Minute, new DateTime(2024, 1, 1, 20, 0, 0), new DateTime(2024, 1, 1, 21, 0, 0)); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDBitfinex, Resolution.Hour, new DateTime(2024, 1, 1, 0, 0, 0), new DateTime(2024, 1, 1, 12, 0, 0)); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Daily, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1)); + } + } + + [TestCaseSource(nameof(HistoricalValidDataTestCases))] + public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Resolution resolution, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, TickType.Trade); + + var downloadResponse = _downloader.Get(parameters).ToList(); + + Assert.IsNotEmpty(downloadResponse); + + Log.Trace($"{symbol}.{resolution}.[{startDateTimeUtc} - {endDateTimeUtc}]: Amount = {downloadResponse.Count}"); + + CoinApiTestHelper.AssertSymbol(downloadResponse.First().Symbol, symbol); + + CoinApiTestHelper.AssertBaseData(downloadResponse, resolution); + } + + private static IEnumerable HistoricalInvalidDataTestCases + { + get + { + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1, 20, 0, 0), new DateTime(2024, 1, 1, 21, 0, 0), TickType.Trade) + .SetDescription($"Not supported - {Resolution.Tick}"); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1), new DateTime(2023, 1, 1), TickType.Trade) + .SetDescription("Wrong startDateTime - startDateTime > endDateTime"); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Quote) + .SetDescription($"Not supported - {TickType.Quote}"); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.OpenInterest) + .SetDescription($"Not supported - {TickType.OpenInterest}"); + } + } + + [TestCaseSource(nameof(HistoricalInvalidDataTestCases))] + public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol, Resolution resolution, DateTime startDateTimeUtc, DateTime endDateTimeUtc, TickType tickType) + { + var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, tickType); + + var downloadResponse = _downloader.Get(parameters).ToList(); + + Assert.IsEmpty(downloadResponse); + } + + private static IEnumerable HistoricalInvalidDataThrowExceptionTestCases + { + get + { + yield return new TestCaseData(Symbol.Create("BTCBTC", SecurityType.Crypto, Market.Binance)) + .SetDescription($"Wrong Symbol - 'BTCBTC'"); + yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Equity, Market.Binance)) + .SetDescription($"Wrong SecurityType - {SecurityType.Equity}"); + } + } + + [TestCaseSource(nameof(HistoricalInvalidDataThrowExceptionTestCases))] + public void DownloadsHistoricalDataWithInvalidDataTestParametersThrowException(Symbol symbol) + { + var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Trade); + + Assert.That(() => _downloader.Get(parameters).ToList(), Throws.Exception); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs index e483e32..b670d3a 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -30,6 +30,8 @@ public void ThrowsOnFailedAuthentication() { using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler(); }); + + Config.Reset(); } } } diff --git a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs new file mode 100644 index 0000000..5f9e336 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs @@ -0,0 +1,78 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Logging; +using QuantConnect.Securities; +using QuantConnect.Data.Market; + +namespace QuantConnect.CoinAPI +{ + public class CoinAPIDataDownloader : IDataDownloader, IDisposable + { + private readonly CoinApiDataQueueHandler _historyProvider; + + private readonly MarketHoursDatabase _marketHoursDatabase; + + public CoinAPIDataDownloader() + { + _historyProvider = new CoinApiDataQueueHandler(); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + } + + public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetParameters) + { + if (dataDownloaderGetParameters.TickType != TickType.Trade) + { + Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}: Not supported data type - {dataDownloaderGetParameters.TickType}. " + + $"Currently available support only for historical of type - {nameof(TickType.Trade)}"); + yield break; + } + + if (dataDownloaderGetParameters.EndUtc < dataDownloaderGetParameters.StartUtc) + { + Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + yield break; + } + + var symbol = dataDownloaderGetParameters.Symbol; + + var historyRequests = new HistoryRequest( + startTimeUtc: dataDownloaderGetParameters.StartUtc, + endTimeUtc: dataDownloaderGetParameters.EndUtc, + dataType: typeof(TradeBar), + symbol: symbol, + resolution: dataDownloaderGetParameters.Resolution, + exchangeHours: _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType), + dataTimeZone: _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType), + fillForwardResolution: dataDownloaderGetParameters.Resolution, + includeExtendedMarketHours: true, + isCustomData: false, + dataNormalizationMode: DataNormalizationMode.Raw, + tickType: TickType.Trade); + + foreach (var slice in _historyProvider.GetHistory(historyRequests)) + { + yield return slice; + } + } + + public void Dispose() + { + _historyProvider.DisposeSafely(); + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs index 849c4a4..e4cda0d 100644 --- a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs @@ -27,6 +27,11 @@ namespace QuantConnect.CoinAPI { public partial class CoinApiDataQueueHandler { + /// + /// Indicates whether the warning for invalid history has been fired. + /// + private bool _invalidHistoryDataTypeWarningFired; + public override void Initialize(HistoryProviderInitializeParameters parameters) { // NOP @@ -54,13 +59,17 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) if (historyRequest.Resolution == Resolution.Tick) { - Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical ticks, only OHLCV timeseries"); + Log.Error($"CoinApiDataQueueHandler.GetHistory(): No historical ticks, only OHLCV timeseries"); yield break; } if (historyRequest.DataType == typeof(QuoteBar)) { - Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical QuoteBars , only TradeBars"); + if (!_invalidHistoryDataTypeWarningFired) + { + Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical QuoteBars , only TradeBars"); + _invalidHistoryDataTypeWarningFired = true; + } yield break; } From 3134a5670acbe2be22a5a6c2dc1935da23ec4a11 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 15:47:14 +0200 Subject: [PATCH 18/34] fix: handle exception when parsing response in History --- QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs index e4cda0d..bafc789 100644 --- a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs @@ -22,6 +22,7 @@ using QuantConnect.CoinAPI.Messages; using QuantConnect.Lean.Engine.DataFeeds; using HistoryRequest = QuantConnect.Data.HistoryRequest; +using QuantConnect.CoinAPI.Models; namespace QuantConnect.CoinAPI { @@ -112,8 +113,17 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) // Log the information associated with the API Key's rest call limits. TraceRestUsage(response); - // Deserialize to array - var coinApiHistoryBars = JsonConvert.DeserializeObject(response.Content); + HistoricalDataMessage[]? coinApiHistoryBars; + try + { + // Deserialize to array + coinApiHistoryBars = JsonConvert.DeserializeObject(response.Content); + } + catch (JsonSerializationException) + { + var error = JsonConvert.DeserializeObject(response.Content); + throw new Exception(error.Error); + } // Can be no historical data for a short period interval if (!coinApiHistoryBars.Any()) From 43cb5ce847b65c028294ec0b394bb8b76d888ea3 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 16:21:36 +0200 Subject: [PATCH 19/34] feat: update Readme --- README.md | 81 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 642fc49..abf0d10 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,60 @@ ![LEAN Data Source SDK](http://cdn.quantconnect.com.s3.us-east-1.amazonaws.com/datasources/Github_LeanDataSourceSDK.png) -# Lean DataSource SDK +# Lean CoinAPI DataSource Plugin [![Build Status](https://github.com/QuantConnect/LeanDataSdk/workflows/Build%20%26%20Test/badge.svg)](https://github.com/QuantConnect/LeanDataSdk/actions?query=workflow%3A%22Build%20%26%20Test%22) ### Introduction -The Lean Data SDK is a cross-platform template repository for developing custom data types for Lean. -These data types will be consumed by [QuantConnect](https://www.quantconnect.com/) trading algorithms and research environment, locally or in the cloud. +Welcome to the CoinAPI Connector Library for .NET 6. This open-source project provides a robust and efficient C# library designed to seamlessly connect with the CoinAPI. The library facilitates easy integration with the QuantConnect [LEAN Algorithmic Trading Engine](https://github.com/quantConnect/Lean), offering a clear and straightforward way for users to incorporate CoinAPI's extensive financial datasets into their algorithmic trading strategies. -It is composed by example .Net solution for the data type and converter scripts. +### CoinAPI Overview +CoinAPI is a reliable provider of real-time and historical financial market data, offering support for traditional asset classes such as cryptocurrencies and crypto futures across various exchanges. With CoinAPI, developers can access a wealth of data to enhance their trading strategies and decision-making processes. -### Prerequisites +### Features -The solution targets dotnet 5, for installation instructions please follow [dotnet download](https://dotnet.microsoft.com/download). +- **Easy Integration:** Simple and intuitive integration process, allowing developers to quickly incorporate CoinAPI's data into their trading algorithms. +- **Rich Financial Data:** Access to a vast array of real-time and historical data for cryptocurrencies and crypto futures, empowering developers to make informed trading decisions. +- **Flexible Configuration:** Customizable settings to tailor the integration according to specific trading needs and preferences. +- **Symbol SecurityType Support:** + - [x] Crypto + - [x] CryptoFuture +- **Exchange Support:** + - [x] COINBASE + - [x] BITFINEX + - [x] BINANCE + - [x] KRAKEN + - [x] BINANCEUS +- **Backtesting and Research:** Seamlessly test and refine your trading algorithms using CoinAPI's data within QuantConnect LEAN's backtesting and research modes, enabling you to optimize your strategies with confidence. -The data downloader and converter script can be developed in different ways: C# executable, Python script, Python Jupyter notebook or even a bash script. -- The python script should be compatible with python 3.6.8 -- Bash script will run on Ubuntu Bionic - -Specifically, the enviroment where these scripts will be run is [quantconnect/research](https://hub.docker.com/repository/docker/quantconnect/research) based on [quantconnect/lean:foundation](https://hub.docker.com/repository/docker/quantconnect/lean). +### Contribute to the Project +Contributions to this open-source project are welcome! If you find any issues, have suggestions for improvements, or want to add new features, please open an issue or submit a pull request. ### Installation - -The "Use this template" feature should be used for each unique data source which requires its own data processing. Once it is cloned locally, you should be able to successfully build the solution, run all tests and execute the downloader and/or conveter scripts. The final version should pass all CI tests of GitHub Actions. - -Once ready, please contact support@quantconnect.com and we will create a listing in the QuantConnect Data Market for your company and link to your public repository and commit hash. - -### Datasets Vendor Requirements - -Key requirements for new vendors include: - - - A well-defined dataset with a clear and static vision for the data to minimize churn or changes as people will be building systems from it. This is easiest with "raw" data (e.g. sunshine hours vs a sentiment algorithm) - - Robust ticker and security links to ensure the tickers are tracked well through time, or accurately point in time. ISIN, FIGI, or point in time ticker supported - - Robust funding to ensure viable for at least 1 year - - Robust API to ensure reliable up-time. No dead links on site or and 502 servers while using API - - Consistent delivery schedule, on time and in time for market trading - - Consistent data format with notifications and lead time on data format updates - - At least 1 year of historical point in time data - - Survivorship bias free data - - Good documentation for the dataset - - -### Tutorials - - - See [Tutorials](https://www.quantconnect.com/docs/v2/our-platform/datasets/contributing-datasets) for a step by step guide for creating a new LEAN Data Source. \ No newline at end of file +To contribute to the CoinAPI Connector Library for .NET 6 within QuantConnect LEAN, follow these steps: +1. **Obtain API Key:** Sign up for a free CoinAPI key [here](https://docs.coinapi.io/) if you don't have one. +2. **Fork the Project:** Fork the repository by clicking the "Fork" button at the top right of the GitHub page. +3. Clone Your Forked Repository: +``` +git clone https://github.com/your-username/Lean.DataSource.CoinAPI.git +``` +4. **Configuration:** + - Set the `coinapi-api-key` in your QuantConnect configuration (config.json or environment variables). + - [optional] Set the `coinapi-product` (by default: Free) +``` +{ + "coinapi-api-key": "", + "coinapi-product": "", +} +``` + +### Price Plan +For detailed information on CoinAPI's pricing plans, please refer to the [CoinAPI Pricing](https://www.coinapi.io/market-data-api/pricing) page. + +### Documentation +Refer to the [documentation](https://www.quantconnect.com/docs/v2/lean-cli/datasets/coinapi) for detailed information on the library's functions, parameters, and usage examples. + +### License +This project is licensed under the MIT License - see the [LICENSE](#) file for details. + +Happy coding and algorithmic trading! \ No newline at end of file From c417baaad467f2b5ef0e6c63d553585d0dff7403 Mon Sep 17 00:00:00 2001 From: Roman Yavnikov <45608740+Romazes@users.noreply.github.com> Date: Tue, 13 Feb 2024 16:24:55 +0200 Subject: [PATCH 20/34] Create LICENSE --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 270a08f69e13835434e759b1ecd1785193ca0b72 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 16:44:35 +0200 Subject: [PATCH 21/34] fix: reset config in test where we change config --- QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs index b670d3a..0eb7221 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -21,6 +21,13 @@ namespace QuantConnect.CoinAPI.Tests [TestFixture] public class CoinApiAdditionalTests { + [TearDown] + public void TearDown() + { + TestSetup t = new(); + t.GlobalSetup(); + } + [Test] public void ThrowsOnFailedAuthentication() { @@ -30,8 +37,6 @@ public void ThrowsOnFailedAuthentication() { using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler(); }); - - Config.Reset(); } } } From 905603b4f5d09741446bad600f4f7629c68820f9 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 17:08:18 +0200 Subject: [PATCH 22/34] refactor: create RestClient at once time --- QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs index bafc789..d052656 100644 --- a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs @@ -28,6 +28,8 @@ namespace QuantConnect.CoinAPI { public partial class CoinApiDataQueueHandler { + private readonly RestClient restClient = new RestClient(); + /// /// Indicates whether the warning for invalid history has been fired. /// @@ -99,16 +101,13 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) var coinApiEndTime = currentEndTime.ToStringInvariant("s"); // Construct URL for rest request - var baseUrl = - "https://rest.coinapi.io/v1/ohlcv/" + + restClient.BaseUrl = new Uri("https://rest.coinapi.io/v1/ohlcv/" + $"{coinApiSymbol}/history?period_id={coinApiPeriod}&limit={HistoricalDataPerRequestLimit}" + - $"&time_start={coinApiStartTime}&time_end={coinApiEndTime}"; + $"&time_start={coinApiStartTime}&time_end={coinApiEndTime}"); - // Execute - var client = new RestClient(baseUrl); var restRequest = new RestRequest(Method.GET); restRequest.AddHeader("X-CoinAPI-Key", _apiKey); - var response = client.Execute(restRequest); + var response = restClient.Execute(restRequest); // Log the information associated with the API Key's rest call limits. TraceRestUsage(response); From 13c406ac30581568bb2fb2a3fd2f952efc877428 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 17:25:59 +0200 Subject: [PATCH 23/34] refactor: create RestRequest only once --- QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs index d052656..6f067ff 100644 --- a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs +++ b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs @@ -30,6 +30,8 @@ public partial class CoinApiDataQueueHandler { private readonly RestClient restClient = new RestClient(); + private readonly RestRequest restRequest = new(Method.GET); + /// /// Indicates whether the warning for invalid history has been fired. /// @@ -105,8 +107,7 @@ public IEnumerable GetHistory(HistoryRequest historyRequest) $"{coinApiSymbol}/history?period_id={coinApiPeriod}&limit={HistoricalDataPerRequestLimit}" + $"&time_start={coinApiStartTime}&time_end={coinApiEndTime}"); - var restRequest = new RestRequest(Method.GET); - restRequest.AddHeader("X-CoinAPI-Key", _apiKey); + restRequest.AddOrUpdateHeader("X-CoinAPI-Key", _apiKey); var response = restClient.Execute(restRequest); // Log the information associated with the API Key's rest call limits. From 62d6fb05dc4cbe8e475b90e8544eb0989c483e94 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 18:35:22 +0200 Subject: [PATCH 24/34] rename: Converter to DataProcessing --- .github/workflows/build.yml | 4 ++-- .gitignore | 2 +- .../CoinApiDataConverter.cs | 3 ++- .../CoinApiDataConverterProgram.cs | 2 +- .../CoinApiDataReader.cs | 2 +- .../CoinApiEntryData.cs | 2 +- .../DataProcessing.csproj | 12 ++++-------- .../Program.cs | 2 +- .../sync.sh | 0 Lean.DataSource.CoinAPI.sln | 2 +- 10 files changed, 14 insertions(+), 17 deletions(-) rename {QuantConnect.CoinAPI.Converter => DataProcessing}/CoinApiDataConverter.cs (99%) rename {QuantConnect.CoinAPI.Converter => DataProcessing}/CoinApiDataConverterProgram.cs (97%) rename {QuantConnect.CoinAPI.Converter => DataProcessing}/CoinApiDataReader.cs (99%) rename {QuantConnect.CoinAPI.Converter => DataProcessing}/CoinApiEntryData.cs (97%) rename QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj => DataProcessing/DataProcessing.csproj (82%) rename {QuantConnect.CoinAPI.Converter => DataProcessing}/Program.cs (97%) rename {QuantConnect.CoinAPI.Converter => DataProcessing}/sync.sh (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd07fab..1a93820 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,8 +49,8 @@ jobs: - name: Build QuantConnect.CoinAPI run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: Build QuantConnect.CoinAPI.Converter - run: dotnet build ./QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Build DataProcessing + run: dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - name: Build QuantConnect.CoinAPI.Tests run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 diff --git a/.gitignore b/.gitignore index 39e099c..7cd3cc7 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,7 @@ *.hex # QC Cloud Setup Bash Files -!QuantConnect.CoinAPI.Converter/*.sh +!DataProcessing/*.sh *.sh # Include docker launch scripts for Mac/Linux !run_docker.sh diff --git a/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs b/DataProcessing/CoinApiDataConverter.cs similarity index 99% rename from QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs rename to DataProcessing/CoinApiDataConverter.cs index c10286f..ccd00e9 100644 --- a/QuantConnect.CoinAPI.Converter/CoinApiDataConverter.cs +++ b/DataProcessing/CoinApiDataConverter.cs @@ -19,8 +19,9 @@ using System.Diagnostics; using QuantConnect.Logging; using QuantConnect.ToolBox; +using QuantConnect.CoinAPI; -namespace QuantConnect.CoinAPI.Converter +namespace QuantConnect.DataProcessing { /// /// Console application for converting CoinApi raw data into Lean data format for high resolutions (tick, second and minute) diff --git a/QuantConnect.CoinAPI.Converter/CoinApiDataConverterProgram.cs b/DataProcessing/CoinApiDataConverterProgram.cs similarity index 97% rename from QuantConnect.CoinAPI.Converter/CoinApiDataConverterProgram.cs rename to DataProcessing/CoinApiDataConverterProgram.cs index dca301d..0609b17 100644 --- a/QuantConnect.CoinAPI.Converter/CoinApiDataConverterProgram.cs +++ b/DataProcessing/CoinApiDataConverterProgram.cs @@ -16,7 +16,7 @@ using System.Globalization; -namespace QuantConnect.CoinAPI.Converter +namespace QuantConnect.DataProcessing { /// /// Coin API Main entry point for ToolBox. diff --git a/QuantConnect.CoinAPI.Converter/CoinApiDataReader.cs b/DataProcessing/CoinApiDataReader.cs similarity index 99% rename from QuantConnect.CoinAPI.Converter/CoinApiDataReader.cs rename to DataProcessing/CoinApiDataReader.cs index 6010c9c..420dec6 100644 --- a/QuantConnect.CoinAPI.Converter/CoinApiDataReader.cs +++ b/DataProcessing/CoinApiDataReader.cs @@ -21,7 +21,7 @@ using QuantConnect.Data.Market; using CompressionMode = System.IO.Compression.CompressionMode; -namespace QuantConnect.CoinAPI.Converter +namespace QuantConnect.DataProcessing { /// /// Reader class for CoinAPI crypto raw data. diff --git a/QuantConnect.CoinAPI.Converter/CoinApiEntryData.cs b/DataProcessing/CoinApiEntryData.cs similarity index 97% rename from QuantConnect.CoinAPI.Converter/CoinApiEntryData.cs rename to DataProcessing/CoinApiEntryData.cs index d10847e..a4d5eb0 100644 --- a/QuantConnect.CoinAPI.Converter/CoinApiEntryData.cs +++ b/DataProcessing/CoinApiEntryData.cs @@ -14,7 +14,7 @@ * limitations under the License. */ -namespace QuantConnect.CoinAPI.Converter +namespace QuantConnect.DataProcessing { /// /// Contains information extracted from CoinAPI entry name diff --git a/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj b/DataProcessing/DataProcessing.csproj similarity index 82% rename from QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj rename to DataProcessing/DataProcessing.csproj index 6f034cd..a5bc5cb 100644 --- a/QuantConnect.CoinAPI.Converter/QuantConnect.CoinAPI.Converter.csproj +++ b/DataProcessing/DataProcessing.csproj @@ -3,21 +3,17 @@ Release AnyCPU - net6.0 - QuantConnect.CoinAPI.Converter - QuantConnect.CoinAPI.Converter - QuantConnect.CoinAPI.Converter - QuantConnect.CoinAPI.Converter Exe + net6.0 + process + true bin\$(Configuration) false - true - false QuantConnect LEAN CoinAPI Converter Data Source: CoinAPI Converter Data Source plugin for Lean enable enable - + full bin\Debug\ diff --git a/QuantConnect.CoinAPI.Converter/Program.cs b/DataProcessing/Program.cs similarity index 97% rename from QuantConnect.CoinAPI.Converter/Program.cs rename to DataProcessing/Program.cs index ed2ac48..5dd4f5c 100644 --- a/QuantConnect.CoinAPI.Converter/Program.cs +++ b/DataProcessing/Program.cs @@ -2,7 +2,7 @@ using System.Globalization; using QuantConnect.Configuration; -namespace QuantConnect.CoinAPI.Converter +namespace QuantConnect.DataProcessing { internal class Program { diff --git a/QuantConnect.CoinAPI.Converter/sync.sh b/DataProcessing/sync.sh similarity index 100% rename from QuantConnect.CoinAPI.Converter/sync.sh rename to DataProcessing/sync.sh diff --git a/Lean.DataSource.CoinAPI.sln b/Lean.DataSource.CoinAPI.sln index bdf8f8e..1c6eb39 100644 --- a/Lean.DataSource.CoinAPI.sln +++ b/Lean.DataSource.CoinAPI.sln @@ -7,7 +7,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI", "Qua EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Converter", "QuantConnect.CoinAPI.Converter\QuantConnect.CoinAPI.Converter.csproj", "{881514B4-641E-4EDC-8020-6BEA0CC8F48C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataProcessing", "DataProcessing\DataProcessing.csproj", "{881514B4-641E-4EDC-8020-6BEA0CC8F48C}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From bfb08d24944eb4e81124fd11fae5c2f8710672b4 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 18:51:30 +0200 Subject: [PATCH 25/34] refactor: test OneTimeSetUp to testing class --- .../CoinAPIHistoryProviderTests.cs | 8 +++++++- .../CoinApiDataQueueHandlerTest.cs | 20 ++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index a30d625..a7c93fd 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -27,7 +27,13 @@ public class CoinAPIHistoryProviderTests { private static readonly Symbol _CoinbaseBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); private static readonly Symbol _BitfinexBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); - private readonly CoinApiDataQueueHandlerMock _coinApiDataQueueHandler = new CoinApiDataQueueHandlerMock(); + private CoinApiDataQueueHandlerMock _coinApiDataQueueHandler; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _coinApiDataQueueHandler = new CoinApiDataQueueHandlerMock(); + } // -- DATA TO TEST -- private static TestCaseData[] TestData => new[] diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index f2a414e..26452f1 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -28,20 +28,30 @@ public class CoinApiDataQueueHandlerTest private CoinApiDataQueueHandler _coinApiDataQueueHandler; private CancellationTokenSource _cancellationTokenSource; - [SetUp] - public void SetUp() + [OneTimeSetUp] + public void OneTimeSetUp() { _coinApiDataQueueHandler = new(); - _cancellationTokenSource = new(); } - [TearDown] - public void TearDown() + [OneTimeTearDown] + public void OneTimeTearDown() { if (_coinApiDataQueueHandler != null) { _coinApiDataQueueHandler.Dispose(); } + } + + [SetUp] + public void SetUp() + { + _cancellationTokenSource = new(); + } + + [TearDown] + public void TearDown() + { _cancellationTokenSource.Dispose(); } From a6bb477f8bed94e86aa442175b3a94a76852cf71 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 18:53:00 +0200 Subject: [PATCH 26/34] refactor: increase delay in DQH tests --- QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 26452f1..6d7927d 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -81,7 +81,7 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() }, () => _cancellationTokenSource.Cancel()); - Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(20), _cancellationTokenSource.Token)); + Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token)); _coinApiDataQueueHandler.Unsubscribe(dataConfig); @@ -130,7 +130,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() }); } - resetEvent.WaitOne(TimeSpan.FromSeconds(30), _cancellationTokenSource.Token); + resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); foreach (var data in symbolBaseData) { @@ -145,7 +145,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() if (dataConfigs.Count != 0) { - resetEvent.WaitOne(TimeSpan.FromSeconds(20), _cancellationTokenSource.Token); + resetEvent.WaitOne(TimeSpan.FromSeconds(30), _cancellationTokenSource.Token); } foreach (var config in dataConfigs) @@ -188,7 +188,7 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() _cancellationTokenSource.Cancel(); }); - resetEvent.WaitOne(TimeSpan.FromSeconds(30), _cancellationTokenSource.Token); + resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); _coinApiDataQueueHandler.Unsubscribe(config); From ba5cb2d1667455d5b4ea5027b3d1682b066f511a Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 19:22:41 +0200 Subject: [PATCH 27/34] fix: change delay and init DQH class tests --- .../CoinApiDataQueueHandlerTest.cs | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 6d7927d..0c7b60c 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -28,24 +28,10 @@ public class CoinApiDataQueueHandlerTest private CoinApiDataQueueHandler _coinApiDataQueueHandler; private CancellationTokenSource _cancellationTokenSource; - [OneTimeSetUp] - public void OneTimeSetUp() - { - _coinApiDataQueueHandler = new(); - } - - [OneTimeTearDown] - public void OneTimeTearDown() - { - if (_coinApiDataQueueHandler != null) - { - _coinApiDataQueueHandler.Dispose(); - } - } - [SetUp] public void SetUp() { + _coinApiDataQueueHandler = new(); _cancellationTokenSource = new(); } @@ -53,6 +39,11 @@ public void SetUp() public void TearDown() { _cancellationTokenSource.Dispose(); + + if (_coinApiDataQueueHandler != null) + { + _coinApiDataQueueHandler.Dispose(); + } } [Test] @@ -190,8 +181,19 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); + // if seq is empty, give additional chance + if (tickData.Count == 0) + { + resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); + } + _coinApiDataQueueHandler.Unsubscribe(config); + if (tickData.Count == 0) + { + Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDFutureTickOnDifferentMarkets)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}"); + } + CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); CoinApiTestHelper.AssertBaseData(tickData, resolution); From 10dc8933c823d6483074d5eedfa6ac3ffa65a8a3 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 21:44:32 +0200 Subject: [PATCH 28/34] refactor: GlobalSetup make static --- QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs | 10 +++------- QuantConnect.CoinAPI.Tests/TestSetup.cs | 4 ++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs index 0eb7221..97ee813 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -21,13 +21,6 @@ namespace QuantConnect.CoinAPI.Tests [TestFixture] public class CoinApiAdditionalTests { - [TearDown] - public void TearDown() - { - TestSetup t = new(); - t.GlobalSetup(); - } - [Test] public void ThrowsOnFailedAuthentication() { @@ -37,6 +30,9 @@ public void ThrowsOnFailedAuthentication() { using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler(); }); + + // reset api key + TestSetup.GlobalSetup(); } } } diff --git a/QuantConnect.CoinAPI.Tests/TestSetup.cs b/QuantConnect.CoinAPI.Tests/TestSetup.cs index f881a82..05bf2fb 100644 --- a/QuantConnect.CoinAPI.Tests/TestSetup.cs +++ b/QuantConnect.CoinAPI.Tests/TestSetup.cs @@ -22,10 +22,10 @@ namespace QuantConnect.CoinAPI.Tests { [SetUpFixture] - public class TestSetup + public static class TestSetup { [OneTimeSetUp] - public void GlobalSetup() + public static void GlobalSetup() { // Log.DebuggingEnabled = true; Log.LogHandler = new CompositeLogHandler(); From 9b6019b55a2b4fd3b0e8ee1d9e70e179ae1e12c6 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 21:45:43 +0200 Subject: [PATCH 29/34] refactor: ProcessFeed in DQH tests --- .../CoinApiDataQueueHandlerTest.cs | 45 +++++++++++++++++-- .../CoinApiTestHelper.cs | 28 ------------ 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 0c7b60c..f3bd9c4 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -16,6 +16,7 @@ using NUnit.Framework; using QuantConnect.Data; +using QuantConnect.Util; using QuantConnect.Logging; using QuantConnect.Data.Market; using System.Collections.Concurrent; @@ -55,8 +56,9 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() var symbol = CoinApiTestHelper.BTCUSDCoinbase; var dataConfig = CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); - CoinApiTestHelper.ProcessFeed( + ProcessFeed( _coinApiDataQueueHandler.Subscribe(dataConfig, (s, e) => { }), + _cancellationTokenSource.Token, tick => { if (tick != null) @@ -79,6 +81,8 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() CoinApiTestHelper.AssertSymbol(tradeBars.First().Symbol, symbol); CoinApiTestHelper.AssertBaseData(tradeBars, resolution); + + _cancellationTokenSource.Cancel(); } [Test] @@ -105,13 +109,14 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() foreach (var config in dataConfigs) { - CoinApiTestHelper.ProcessFeed( + ProcessFeed( _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), + _cancellationTokenSource.Token, tick => { if (tick != null) { - Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); symbolBaseData[tick.Symbol].Add(tick); } }, @@ -148,6 +153,8 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() { CoinApiTestHelper.AssertBaseData(data, resolution); } + + _cancellationTokenSource.Cancel(); } [Test] @@ -159,8 +166,9 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() var symbol = CoinApiTestHelper.BTCUSDTFutureBinance; var config = CoinApiTestHelper.GetSubscriptionTickDataConfigs(symbol); - CoinApiTestHelper.ProcessFeed( + ProcessFeed( _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), + _cancellationTokenSource.Token, tick => { if (tick != null) @@ -197,6 +205,35 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); CoinApiTestHelper.AssertBaseData(tickData, resolution); + + _cancellationTokenSource.Cancel(); + } + + private Task ProcessFeed(IEnumerator enumerator, CancellationToken cancellationToken, Action? callback = null, Action? throwExceptionCallback = null) + { + return Task.Factory.StartNew(() => + { + try + { + while (enumerator.MoveNext() && !cancellationToken.IsCancellationRequested) + { + BaseData tick = enumerator.Current; + callback?.Invoke(tick); + Thread.Sleep(100); + } + } + catch + { + throw; + } + }, cancellationToken).ContinueWith(task => + { + if (throwExceptionCallback != null) + { + throwExceptionCallback(); + } + Log.Error("The throwExceptionCallback is null."); + }, TaskContinuationOptions.OnlyOnFaulted); } } } diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs index 29a88b9..a06bf6c 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -74,34 +74,6 @@ public static void AssertBaseData(List tradeBars, Resolution expectedR } } - public static void ProcessFeed(IEnumerator enumerator, Action? callback = null, Action? throwExceptionCallback = null) - { - Task.Factory.StartNew(() => - { - try - { - while (enumerator.MoveNext()) - { - BaseData tick = enumerator.Current; - callback?.Invoke(tick); - - Thread.Sleep(1000); - } - } - catch - { - throw; - } - }).ContinueWith(task => - { - if (throwExceptionCallback != null) - { - throwExceptionCallback(); - } - Log.Error("The throwExceptionCallback is null."); - }, TaskContinuationOptions.OnlyOnFaulted); - } - public static SubscriptionDataConfig GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) { return GetSubscriptionDataConfig(symbol, resolution); From 8586531af73923c7e332ca6321aa63db5774739e Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Feb 2024 23:31:30 +0200 Subject: [PATCH 30/34] feat: add some explicit and log trace --- .../CoinAPIDataDownloaderTests.cs | 2 +- .../CoinAPIHistoryProviderTests.cs | 2 +- QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs | 2 +- .../CoinApiDataQueueHandlerTest.cs | 10 ++++++++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs index 669d48a..6b90538 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -19,7 +19,7 @@ namespace QuantConnect.CoinAPI.Tests { - [TestFixture] + [TestFixture, Explicit("")] public class CoinAPIDataDownloaderTests { private CoinAPIDataDownloader _downloader; diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index a7c93fd..102c038 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -22,7 +22,7 @@ namespace QuantConnect.CoinAPI.Tests { - [TestFixture] + [TestFixture, Explicit("")] public class CoinAPIHistoryProviderTests { private static readonly Symbol _CoinbaseBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); diff --git a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs index 7303fc1..b1a8fc0 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs @@ -18,7 +18,7 @@ namespace QuantConnect.CoinAPI.Tests { - [TestFixture] + [TestFixture, Explicit("")] public class CoinAPISymbolMapperTests { private CoinApiSymbolMapper _coinApiSymbolMapper; diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index f3bd9c4..7741676 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -63,7 +63,8 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() { if (tick != null) { - Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); tradeBars.Add(tick); if (tradeBars.Count > 5) @@ -117,6 +118,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() if (tick != null) { // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); symbolBaseData[tick.Symbol].Add(tick); } }, @@ -173,7 +175,8 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() { if (tick != null) { - Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + //Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); tickData.Add(tick); if (tickData.Count > 5) @@ -218,6 +221,9 @@ private Task ProcessFeed(IEnumerator enumerator, CancellationToken can while (enumerator.MoveNext() && !cancellationToken.IsCancellationRequested) { BaseData tick = enumerator.Current; + + Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); + callback?.Invoke(tick); Thread.Sleep(100); } From be8cd7d87e2cc3af16b810bd4e3cc6c25a645cee Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Feb 2024 00:06:55 +0200 Subject: [PATCH 31/34] refactor: validation on null tick remove: thread sleep --- .../CoinApiDataQueueHandlerTest.cs | 47 ++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 7741676..1677cbf 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -61,16 +61,13 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() _cancellationTokenSource.Token, tick => { - if (tick != null) - { - // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); - Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); - tradeBars.Add(tick); + // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); + tradeBars.Add(tick); - if (tradeBars.Count > 5) - { - resetEvent.Set(); - } + if (tradeBars.Count > 5) + { + resetEvent.Set(); } }, () => _cancellationTokenSource.Cancel()); @@ -115,12 +112,9 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() _cancellationTokenSource.Token, tick => { - if (tick != null) - { - // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); - Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); - symbolBaseData[tick.Symbol].Add(tick); - } + // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); + symbolBaseData[tick.Symbol].Add(tick); }, () => { @@ -173,16 +167,13 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() _cancellationTokenSource.Token, tick => { - if (tick != null) - { - //Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); - Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); - tickData.Add(tick); + //Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); + Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); + tickData.Add(tick); - if (tickData.Count > 5) - { - resetEvent.Set(); - } + if (tickData.Count > 5) + { + resetEvent.Set(); } }, () => @@ -193,7 +184,7 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); // if seq is empty, give additional chance - if (tickData.Count == 0) + if (tickData.Count == 0) { resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); } @@ -224,8 +215,10 @@ private Task ProcessFeed(IEnumerator enumerator, CancellationToken can Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); - callback?.Invoke(tick); - Thread.Sleep(100); + if (tick != null) + { + callback?.Invoke(tick); + } } } catch From 0e92054acc51f6916d93535508180d81c409d16d Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Feb 2024 00:35:32 +0200 Subject: [PATCH 32/34] feat: add delay in ProcessFeed by cancellationToken refactor: future test --- .../CoinApiDataQueueHandlerTest.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 1677cbf..7494453 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -154,13 +154,13 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() } [Test] - public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() + public void SubscribeToBTCUSDTFutureSecondBinance() { var resetEvent = new AutoResetEvent(false); - var resolution = Resolution.Tick; + var resolution = Resolution.Second; var tickData = new List(); var symbol = CoinApiTestHelper.BTCUSDTFutureBinance; - var config = CoinApiTestHelper.GetSubscriptionTickDataConfigs(symbol); + var config = CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); ProcessFeed( _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), @@ -193,7 +193,7 @@ public void SubscribeToBTCUSDFutureTickOnDifferentMarkets() if (tickData.Count == 0) { - Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDFutureTickOnDifferentMarkets)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}"); + Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}"); } CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); @@ -213,12 +213,12 @@ private Task ProcessFeed(IEnumerator enumerator, CancellationToken can { BaseData tick = enumerator.Current; - Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); - if (tick != null) { callback?.Invoke(tick); } + + cancellationToken.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(500)); } } catch From aea3b12a98174602ba48700779e071e49c647c93 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Feb 2024 01:03:52 +0200 Subject: [PATCH 33/34] fix: tick symbol in CryptoFuture test --- QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs | 6 +++--- QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 7494453..474b46c 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -47,7 +47,7 @@ public void TearDown() } } - [Test] + [Test, Explicit("")] public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() { var resetEvent = new AutoResetEvent(false); @@ -83,7 +83,7 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() _cancellationTokenSource.Cancel(); } - [Test] + [Test, Explicit("")] public void SubscribeToBTCUSDSecondOnDifferentMarkets() { var resetEvent = new AutoResetEvent(false); @@ -159,7 +159,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance() var resetEvent = new AutoResetEvent(false); var resolution = Resolution.Second; var tickData = new List(); - var symbol = CoinApiTestHelper.BTCUSDTFutureBinance; + var symbol = CoinApiTestHelper.BTCUSDFutureBinance; var config = CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); ProcessFeed( diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs index a06bf6c..849a1c8 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -32,7 +32,7 @@ public static class CoinApiTestHelper /// /// PERPETUAL BTCUSDT /// - public static readonly Symbol BTCUSDTFutureBinance = Symbol.Create("BTCUSDT", SecurityType.CryptoFuture, Market.Binance); + public static readonly Symbol BTCUSDFutureBinance = Symbol.Create("BTCUSD", SecurityType.CryptoFuture, Market.Binance); public static void AssertSymbol(Symbol actualSymbol, Symbol expectedSymbol) { From 1e4dfae463b53a6907d98b9a82123a10783e1a69 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Feb 2024 01:16:03 +0200 Subject: [PATCH 34/34] remove: Explicit attribute in tests --- .../CoinAPIDataDownloaderTests.cs | 2 +- .../CoinAPIHistoryProviderTests.cs | 2 +- .../CoinAPISymbolMapperTests.cs | 2 +- .../CoinApiDataQueueHandlerTest.cs | 13 +++++-------- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs index 6b90538..669d48a 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -19,7 +19,7 @@ namespace QuantConnect.CoinAPI.Tests { - [TestFixture, Explicit("")] + [TestFixture] public class CoinAPIDataDownloaderTests { private CoinAPIDataDownloader _downloader; diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs index 102c038..a7c93fd 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -22,7 +22,7 @@ namespace QuantConnect.CoinAPI.Tests { - [TestFixture, Explicit("")] + [TestFixture] public class CoinAPIHistoryProviderTests { private static readonly Symbol _CoinbaseBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); diff --git a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs index b1a8fc0..7303fc1 100644 --- a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs +++ b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs @@ -18,7 +18,7 @@ namespace QuantConnect.CoinAPI.Tests { - [TestFixture, Explicit("")] + [TestFixture] public class CoinAPISymbolMapperTests { private CoinApiSymbolMapper _coinApiSymbolMapper; diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs index 474b46c..c63d45a 100644 --- a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -47,7 +47,7 @@ public void TearDown() } } - [Test, Explicit("")] + [Test] public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() { var resetEvent = new AutoResetEvent(false); @@ -61,8 +61,7 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() _cancellationTokenSource.Token, tick => { - // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); - Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}"); tradeBars.Add(tick); if (tradeBars.Count > 5) @@ -83,7 +82,7 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() _cancellationTokenSource.Cancel(); } - [Test, Explicit("")] + [Test] public void SubscribeToBTCUSDSecondOnDifferentMarkets() { var resetEvent = new AutoResetEvent(false); @@ -112,8 +111,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets() _cancellationTokenSource.Token, tick => { - // Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); - Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}"); symbolBaseData[tick.Symbol].Add(tick); }, () => @@ -167,8 +165,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance() _cancellationTokenSource.Token, tick => { - //Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}: tick: {tick}"); - Log.Trace($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(ProcessFeed)}: tick {tick}"); + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}"); tickData.Add(tick); if (tickData.Count > 5)