/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System.IO;
using Ionic.Zip;
using System.Linq;
using QuantConnect.Util;
using QuantConnect.Logging;
using QuantConnect.Interfaces;
using System.Collections.Generic;
namespace QuantConnect.Data
{
///
/// Simple data cache provider, writes and reads directly from disk
/// Used as default for
///
public class DiskDataCacheProvider : IDataCacheProvider
{
private readonly KeyStringSynchronizer _synchronizer;
///
/// Property indicating the data is temporary in nature and should not be cached.
///
public bool IsDataEphemeral => false;
///
/// Creates a new instance
///
public DiskDataCacheProvider() : this(new KeyStringSynchronizer())
{
}
///
/// Creates a new instance using the given synchronizer
///
/// The synchronizer instance to use
public DiskDataCacheProvider(KeyStringSynchronizer locker)
{
_synchronizer = locker;
}
///
/// Fetch data from the cache
///
/// A string representing the key of the cached data
/// An of the cached data
public Stream Fetch(string key)
{
LeanData.ParseKey(key, out var filePath, out var entryName);
return _synchronizer.Execute(filePath, () =>
{
if (!File.Exists(filePath))
{
return null;
}
try
{
using (var zip = ZipFile.Read(filePath))
{
ZipEntry entry;
if (entryName.IsNullOrEmpty())
{
// Return the first entry
entry = zip[0];
}
else
{
// Attempt to find our specific entry
if (!zip.ContainsEntry(entryName))
{
return null;
}
entry = zip[entryName];
}
// Extract our entry and return it
var stream = new MemoryStream();
entry.Extract(stream);
stream.Position = 0;
return stream;
}
}
catch (ZipException exception)
{
Log.Error("DiskDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception);
return null;
}
});
}
///
/// Store the data in the cache. Not implemented in this instance of the IDataCacheProvider
///
/// The source of the data, used as a key to retrieve data in the cache
/// The data as a byte array
public void Store(string key, byte[] data)
{
LeanData.ParseKey(key, out var filePath, out var entryName);
_synchronizer.Execute(filePath, singleExecution: false, () =>
{
Compression.ZipCreateAppendData(filePath, entryName, data, true);
});
}
///
/// Returns a list of zip entries in a provided zip file
///
public List GetZipEntries(string zipFile)
{
return _synchronizer.Execute(zipFile, () =>
{
using var stream = new FileStream(FileExtension.ToNormalizedPath(zipFile), FileMode.Open, FileAccess.Read);
return Compression.GetZipEntryFileNames(stream).ToList();
});
}
///
/// Dispose for this class
///
public void Dispose()
{
//NOP
}
}
}