/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ using System.IO; using Ionic.Zip; using System.Linq; using QuantConnect.Util; using QuantConnect.Logging; using QuantConnect.Interfaces; using System.Collections.Generic; namespace QuantConnect.Data { /// /// Simple data cache provider, writes and reads directly from disk /// Used as default for /// public class DiskDataCacheProvider : IDataCacheProvider { private readonly KeyStringSynchronizer _synchronizer; /// /// Property indicating the data is temporary in nature and should not be cached. /// public bool IsDataEphemeral => false; /// /// Creates a new instance /// public DiskDataCacheProvider() : this(new KeyStringSynchronizer()) { } /// /// Creates a new instance using the given synchronizer /// /// The synchronizer instance to use public DiskDataCacheProvider(KeyStringSynchronizer locker) { _synchronizer = locker; } /// /// Fetch data from the cache /// /// A string representing the key of the cached data /// An of the cached data public Stream Fetch(string key) { LeanData.ParseKey(key, out var filePath, out var entryName); return _synchronizer.Execute(filePath, () => { if (!File.Exists(filePath)) { return null; } try { using (var zip = ZipFile.Read(filePath)) { ZipEntry entry; if (entryName.IsNullOrEmpty()) { // Return the first entry entry = zip[0]; } else { // Attempt to find our specific entry if (!zip.ContainsEntry(entryName)) { return null; } entry = zip[entryName]; } // Extract our entry and return it var stream = new MemoryStream(); entry.Extract(stream); stream.Position = 0; return stream; } } catch (ZipException exception) { Log.Error("DiskDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception); return null; } }); } /// /// Store the data in the cache. Not implemented in this instance of the IDataCacheProvider /// /// The source of the data, used as a key to retrieve data in the cache /// The data as a byte array public void Store(string key, byte[] data) { LeanData.ParseKey(key, out var filePath, out var entryName); _synchronizer.Execute(filePath, singleExecution: false, () => { Compression.ZipCreateAppendData(filePath, entryName, data, true); }); } /// /// Returns a list of zip entries in a provided zip file /// public List GetZipEntries(string zipFile) { return _synchronizer.Execute(zipFile, () => { using var stream = new FileStream(FileExtension.ToNormalizedPath(zipFile), FileMode.Open, FileAccess.Read); return Compression.GetZipEntryFileNames(stream).ToList(); }); } /// /// Dispose for this class /// public void Dispose() { //NOP } } }