QuantConnect
Implement default custom data json reader
Expected Behavior
Either the entire file contents or a proper json fragment is sent to the Reader method or the documentation needs to be updated here to not show the formatted json fragment like that. Hopefully, that's not the final solution. For it to work with the current design, the json needs to be:
[
{ "Date": "20170704", "Symbols": ["SPY", "QQQ", "FB", "AAPL", "IWM"] },
{ "Date": "20170706", "Symbols": ["QQQ", "AAPL", "IWM", "FB", "GOOGL"] },
...
{ "Date": "20170801", "Symbols": ["QQQ", "FB", "AAPL", "IWM", "GOOGL"] },
{ "Date": "20170802", "Symbols": ["QQQ", "IWM", "FB", "BAC", "GOOGL"] }
]
Actual Behavior
Data is sent one line at a time.
Potential Solution
It seems like CollectionSubscriptionDataSourceReader does try to read data one line a time as indicated here. It may be nice to introduce a new FileFormat (i.e. Json, JsonArray, JsonObject) or add a new field to SubscriptionDataSource to some how indicate the whole file should be read.
Reproducing the Problem
Try using algo and data class from here.
System Information
windows, visual studio
Checklist
- [x] I have completely filled out this template
- [x] I have confirmed that this issue exists on the current
masterbranch - [x] I have confirmed that this is not a duplicate issue by searching issues
- [x] I have provided detailed steps to reproduce the issue
a quick note, the above json doesn't work great either when trying to use something like var json = JObject.Parse(line); unless you trim off the comma at the end or you place the comma on a separate line.
Here's an example that was given to me by Mia after quite a few tweaks. The selector function is never entered.
/// <summary>
/// Custom BaseData type to model combo info loaded from object store JSON files.
/// Supports both single-object and array-in-file, using FileFormat.UnfoldingCollection.
/// </summary>
public class ComboInfoUniverseData : BaseData
{
public decimal Price { get; set; }
public decimal Strike { get; set; }
public string Right { get; set; }
public DateTime Expiry { get; set; }
public string ComboName { get; set; }
public Dictionary<string, object> ExtraFields { get; set; } = new();
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
// Each config.Symbol.Value is the file key (e.g., comboinfo1, comboinfo20250704, etc)
var filename = config.Symbol.Value + ".json";
return new SubscriptionDataSource(
filename,
SubscriptionTransportMedium.ObjectStore,
FileFormat.UnfoldingCollection
);
}
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
var result = new BaseDataCollection(date, config.Symbol);
try
{
var json = JObject.Parse(line.Trim(','));
string rawSymbol = json["Symbol"]?.ToString();
if (string.IsNullOrEmpty(rawSymbol))
return null;
Symbol comboSymbol;
// Recognize option tickers of the format "SPY600C20250704"
var optionMatch = System.Text.RegularExpressions.Regex.Match(
rawSymbol,
@"^(?<underlying>[A-Z]+)(?<strike>\d+)(?<right>C|P)(?<expiry>\d{8})$"
);
if (optionMatch.Success)
{
// Parse components from the ticker string
string underlying = optionMatch.Groups["underlying"].Value;
decimal strike = Convert.ToDecimal(optionMatch.Groups["strike"].Value);
OptionRight right = optionMatch.Groups["right"].Value == "C" ? OptionRight.Call : OptionRight.Put;
DateTime expiry = DateTime.ParseExact(optionMatch.Groups["expiry"].Value, "yyyyMMdd", null);
// Create the correct Symbol for this option
comboSymbol = Symbol.CreateOption(
underlying,
Market.USA,
OptionStyle.American,
right,
strike,
expiry
);
}
else
{
// Fallback: treat as equity symbol
comboSymbol = Symbol.Create(rawSymbol, SecurityType.Equity, Market.USA);
}
var item= new ComboInfoUniverseData()
{
Symbol = comboSymbol, // the correct option or equity symbol object
Time = date,
Price = json["Price"] != null ? Convert.ToDecimal(json["Price"]) : 0m,
Value = json["Price"] != null ? Convert.ToDecimal(json["Price"]) : 0m
// ...populate other fields as necessary
};
result.Add(item);
}
catch(Exception ex)
{
// Handle parsing errors gracefully
Trace.WriteLine($"Error parsing combo data: {ex.Message}");
Trace.WriteLine($"Line content: {line}");
return null;
}
return result;
}
}
/// <summary>
/// Example usage: initialize multiple combo universes, each mapped to an object store file (dynamic or static).
/// Universe selector simply picks all valid combo symbols for consideration.
/// </summary>
public class ComboInfoUniverseAlgorithm : QCAlgorithm
{
// Example: list of known files. In practice, you could enumerate them or store list in config/objectstore.
private readonly List<string> _comboFiles = new()
{
"comboinfo1",
"comboinfo20250704",
// "comboinfo20250801"
};
public override void Initialize()
{
SetStartDate(2021, 1, 1);
SetEndDate(2022, 1, 1);
SetCash(100000);
// Loop through files and set up universes accordingly
foreach (var fileKey in this._comboFiles)
{
var symbol = AddData<ComboInfoUniverseData>(fileKey, Resolution.Daily).Symbol;
AddUniverse<ComboInfoUniverseData>(symbol, Resolution.Daily, SelectComboSymbols);
}
}
// Selects all combo symbols from loaded data. Can add filter here.
private IEnumerable<Symbol> SelectComboSymbols(IEnumerable<BaseData> data)
{
foreach (var item in data)
{
if (item is ComboInfoUniverseData combo && !string.IsNullOrEmpty(combo.ComboName))
{
yield return combo.Symbol;
}
}
}
}
// EXAMPLE JSON STRUCTURE for a file "comboinfo20250704.json" (one combo):
/*
{
"Symbol": "SPY_600C_20250704",
"Price": 7.5,
"Strike": 600,
"Right": "C",
"Expiry": "2025-07-04T00:00:00",
"ComboName": "July25C600"
}
*/
// ...or for multiple combos in one file
/*
[
{
"Symbol": "SPY_590C_20250704",
"Price": 12.4,
"Strike": 590,
"Right": "C",
"Expiry": "2025-07-04T00:00:00",
"ComboName": "July25C590"
},
{
"Symbol": "SPY_600C_20250704",
"Price": 7.5,
"Strike": 600,
"Right": "C",
"Expiry": "2025-07-04T00:00:00",
"ComboName": "July25C600"
}
]
*/
// ----------------------------------------
// GUIDELINES FOR MULTIPLE/NEW FILES:
//
// - When you want to add a new combo, upload a new JSON file (with the .json suffix omitted in AddData) to the Object Store. Name could reflect the content ("comboinfoIBM20250801", etc).
// - In Initialize, create a list of file keys and AddData/AddUniverse for each one.
// - To add new combos while the algo is running: add a new file and update the _comboFiles collection (or, in advanced versions, re-enumerate keys and call AddData/AddUniverse for newly discovered keys).
// - Universes refresh per time-slice; new combos will be incorporated as universes are refreshed or data loaded.
// - Use smaller, modular files for flexibility (easier to append, upload, manage combos individually); use bigger files for static batches.
// - JSON lets you add extra fields easily for future features.
//
// Robust, scalable, and clear for universe management!
modified json for simple and multiple:
[
{ "Symbol": "SPY590C20250704", "UnderlyingSymbol": "SPY", "Price": 12.4, "Strike": 590, "Right": "C", "Expiry": "2025-08-08T00:00:00", "ComboName": "July25C590" },
{ "Symbol": "SPY600C20250704", "UnderlyingSymbol": "SPY", "Price": 7.5, "Strike": 600, "Right": "C", "Expiry": "2025-08-08T00:00:00", "ComboName": "July25C600" }
]
{ "Symbol": "SPY600C20250704", "UnderlyingSymbol": "SPY", "Price": 7.5, "Strike": 600, "Right": "C", "Expiry": "2025-08-08T00:00:00", "ComboName": "July25C600"}
Hey @powerdude! Thank you for the detailed report. This is a bug in the documentation, we will work on it. Although ideally I think there's an opportunity here to implement a default json data reader implementation in the lean engine: if there's no custom/override reader implementation, format is json, we can try a simple json deserialization, solving any required json handling, like 1 line, mulitple, jarray, jobject, etc
