C#开源磁盘/内存缓存引擎

前言

昨天写了个 《基于STSdb和fastJson的磁盘/内存缓存》，大家可以先看看。下午用到业务系统时候，觉得可以改进一下，昨晚想了一个晚上，刚才重新实现一下。

更新

1. 增加了对批量处理的支持，写操作速度提升5倍，读操作提升100倍

2. 增加了一个存储provider，可以选择不用STSdb做存储，而用物理文件/Dictionary。

3. 增加了空间回收

4. 增加了对并发的支持

需求

业务系统用的是数据库，数据量大，部分只读或相对稳定业务查询复杂，每次页面加载都要花耗不少时间(不讨论异步)，觉得可以做一下高速缓存，譬如用nosql那种key/value快速存取结果

目的

提供一个简单易用的解决缓存方案，可以根据数据的大小缓存到内存或者磁盘。

实现

存取

方法1. 基于STSdb，提供高效的Key/Value存取，支持磁盘/内存，对Key无限制

方法2. 基于直接物理文件/Dictionary。Key必须是基本类型，譬如int/long/uint/ulong/DateTime/string等。

代码

代码比较简单，花了2个小时写的，很多情况没考虑，譬如磁盘空间/内存不足，自动回收过期缓存等，这些留给大家做家庭作业吧。另外，为了发布方便，STSdb和fastJson的代码都合并到一个项目里。

BaseCahce.cs

这是一个抽象基类，提供存取接口。

using System;using System.Collections.Generic;using System.Linq;using System.Text;namespace Com.SuperCache.Engine{public abstract class BaseCache{protected internal const string KeyExpiration = "Expiration";public abstract void Add<K>(string Category, K Key, object Data);public abstract void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate);public abstract void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate);public abstract List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys);public abstract V Get<K, V>(string Category, K Key);public abstract void Recycle<K>( string Category, long Count);}}

CahceEngine.cs

主要调用缓存引擎

using System;using System.Collections.Generic;using System.Linq;using System.Text;using STSdb4.Database;using fastJSON;using System.IO;namespace Com.SuperCache.Engine{public enum CacheProviders{Default = 1,Raw = 2}public enum RecycleAlgorithms{None = 0,//LRU = 1,MRU = 2}public enum RecycleModes{None = 0,Passive = 1,Active = 2}public class CacheEngine{private BaseCache cacheProvider = null;public CacheEngine(string DataPath): this(CacheProviders.Default, DataPath, RecycleAlgorithms.None, 0, 0, RecycleModes.None){}public CacheEngine(CacheProviders Provider, string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode){switch (Provider){case CacheProviders.Default:cacheProvider = new STSdbCache(DataPath, RecycleAlgorithm, MaxCount, Threshold, RecycleMode);break;case CacheProviders.Raw:cacheProvider = new RawCache(DataPath, RecycleAlgorithm, MaxCount, Threshold, RecycleMode);break;default:break;}}public void Add<K>(string Category, K Key, object Data){cacheProvider.Add<K>(Category, Key, Data);}public void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate){cacheProvider.Add<K, V>(Category, Items, ExpirationDate);}public void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate){cacheProvider.Add<K>(Category, Key, Data, ExpirationDate);}public List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys){return cacheProvider.Get<K, V>(Category, Keys);}public V Get<K, V>(string Category, K Key){return cacheProvider.Get<K, V>(Category, Key);}}}

STSdbCache.cs

STSdb存储引擎

using System;using System.Collections.Generic;using System.Linq;using System.Text;using STSdb4.Database;using fastJSON;using System.IO;namespace Com.SuperCache.Engine{public class STSdbCache : BaseCache{private const string UsageStat = "SuperCacheUsageStat";private string dataPath;private static IStorageEngine memoryInstance = null;private static object syncRoot = new object();private bool isMemory = false;private RecycleAlgorithms recycleAlgorithm;private int maxCount;private int threshold;private RecycleModes recycleMode;public STSdbCache(string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode){dataPath = DataPath;if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString()))dataPath += Path.DirectorySeparatorChar;isMemory = string.IsNullOrEmpty(DataPath);recycleAlgorithm = RecycleAlgorithm;maxCount = MaxCount;threshold = Threshold;recycleMode = RecycleMode;}public override void Add<K>(string Category, K Key, object Data){Add(Category, Key, Data, null);}private IStorageEngine Engine{get{if (isMemory){lock (syncRoot){if (memoryInstance == null)memoryInstance = STSdb.FromMemory();}return memoryInstance;}elsereturn STSdb.FromFile(GetFile(false), GetFile(true));}}private string GetExpirationTable(string Category){return KeyExpiration + "_" + Category;}public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate){long count = 0;lock (syncRoot){var engine = Engine;var table = engine.OpenXIndex<K, string>(Category);var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category));//track recycleIIndex<K, int> usage = null;if (recycleAlgorithm != RecycleAlgorithms.None)usage = engine.OpenXIndex<K, int>(UsageStat);Items.ForEach(i =>{var key = i.Key;var data = i.Value;//will only serialize object other than stringvar result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data);table[key] = result;table.Flush();//specify expiration//default 30 mins to expire from nowvar expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate;expiration[key] = expirationDate;expiration.Flush();//track recycleif (usage != null){usage[key] = 0;if (recycleMode == RecycleModes.Active)Recycle<K>(Category, usage.Count());}});if (usage != null)count = usage.Count();mit();//only dispose disk-based engineif (!isMemory)engine.Dispose();}if (recycleMode == RecycleModes.Passive)Recycle<K>(Category, count);}public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate){Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate);}private string GetFile(bool IsData){if (!Directory.Exists(dataPath))Directory.CreateDirectory(dataPath);return dataPath + "SuperCache." + (IsData ? "dat" : "sys");}public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys){var result = new List<KeyValuePair<K, V>>();lock (syncRoot){var engine = Engine;var table = engine.OpenXIndex<K, string>(Category);var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category));var isCommitRequired = false;//track recycleIIndex<K, int> usage = null;if (recycleAlgorithm != RecycleAlgorithms.None)usage = engine.OpenXIndex<K, int>(UsageStat);Keys.ForEach(key =>{string buffer;V value;if (table.TryGet(key, out buffer)){//will only deserialize object other than stringvalue = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer);bool needUpdate = true;DateTime expirationDate;//get expiration dateif (expiration.TryGet(key, out expirationDate)){//expiredif (expirationDate < DateTime.Now){value = default(V);table.Delete(key);table.Flush();expiration.Delete(key);expiration.Flush();isCommitRequired = true;needUpdate = false;}}//track recycleif (usage != null && needUpdate){usage[key]++;isCommitRequired = true;}}elsevalue = default(V);result.Add(new KeyValuePair<K, V>(key, value));});//only need to commit write actionsif (isCommitRequired)mit();//only dispose disk-based engineif (!isMemory)engine.Dispose();}return result;}public override V Get<K, V>(string Category, K Key){var buffer = Get<K, V>(Category, new K[] { Key });var result = buffer.FirstOrDefault();return result.Value;}public override void Recycle<K>(string Category, long Count){if (Count < maxCount)return;switch (recycleAlgorithm){case RecycleAlgorithms.MRU:lock (syncRoot){var engine = Engine;var table = engine.OpenXIndex<K, string>(Category);var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category));var usage = engine.OpenXIndex<K, int>(UsageStat);//find out expired itemsvar expired = expiration.Where(e => e.Value < DateTime.Now);expired.ForEach(e =>{table.Delete(e.Key);expiration.Delete(e.Key);usage.Delete(e.Key);});//find out least used itemsvar leastUsed = usage.OrderByDescending(s => s.Value).Skip(maxCount - threshold);leastUsed.ForEach(u =>{table.Delete(u.Key);expiration.Delete(u.Key);usage.Delete(u.Key);});table.Flush();expiration.Flush();usage.Flush();mit();if (!isMemory)engine.Dispose();}break;default:break;}}}}

RawCache.cs

物理文件/Dictionary引擎

using System;using System.Collections.Generic;using System.Linq;using System.Text;using STSdb4.Database;using fastJSON;using System.IO;namespace Com.SuperCache.Engine{public class RawCache : BaseCache{private const string ExpirationFileExtension = "exp";private const string DataFileExtension = "dat";private const string statFile = "SuperCache.sta";private string dataPath;private static Dictionary<string, object> memoryData = new Dictionary<string, object>();private static Dictionary<string, DateTime?> memoryExpiration = new Dictionary<string, DateTime?>();private static object syncRoot = new object();private bool isMemory = false;private RecycleAlgorithms recycleAlgorithm;private int maxCount;private int threshold;private static Dictionary<string, KeyValue> usageStat = new Dictionary<string, KeyValue>();private Dictionary<string, KeyValuePair<DateTime, string>> expiredFiles = new Dictionary<string, KeyValuePair<DateTime, string>>();private RecycleModes recycleMode;public RawCache(string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode){dataPath = DataPath;if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString()))dataPath += Path.DirectorySeparatorChar;isMemory = string.IsNullOrEmpty(DataPath);recycleAlgorithm = RecycleAlgorithm;maxCount = MaxCount;threshold = Threshold;recycleMode = RecycleMode;}public override void Add<K>(string Category, K Key, object Data){Add(Category, Key, Data, null);}private string GetExpirationTable(string Category){return KeyExpiration + "_" + Category;}public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate){long count = 0;lock (syncRoot){Items.ForEach(i =>{var key = i.Key;var data = i.Value;var cacheKey = GetKey(Category, key.ToString());if (isMemory){memoryData[cacheKey] = data;memoryExpiration[cacheKey] = ExpirationDate;//recycle algoswitch (recycleAlgorithm){case RecycleAlgorithms.MRU:usageStat[cacheKey] = new KeyValue(string.Empty, 0);if (recycleMode == RecycleModes.Active)Recycle<K>(Category, memoryData.Count);break;default:break;}}else{//will only serialize object other than stringvar result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data);var fileKey = key.ToString();var dataFile = GetFile(Category, fileKey, true);bool exists = File.Exists(dataFile);File.WriteAllText(dataFile, result);//specify expiration//default 30 mins to expire from nowvar expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate;var expirationFile = GetFile(Category, fileKey, false);File.WriteAllText(expirationFile, expirationDate.ToString());//recycle algoif (recycleAlgorithm != RecycleAlgorithms.None){var statFilePath = dataPath + statFile;if (File.Exists(statFilePath)){var buffer = File.ReadAllText(statFilePath);count = Convert.ToInt32(buffer);}if (!exists){count++;File.WriteAllText(statFilePath, count.ToString());}switch (recycleAlgorithm){case RecycleAlgorithms.MRU:usageStat[cacheKey] = new KeyValue(expirationFile, 0);expiredFiles[cacheKey] = new KeyValuePair<DateTime, string>(expirationDate, expirationFile);if (recycleMode == RecycleModes.Active)Recycle<K>(Category, count);break;default:break;}}}});if (recycleAlgorithm != RecycleAlgorithms.None && recycleMode == RecycleModes.Passive){if (isMemory)count = memoryData.Count;Recycle<K>(Category, count);}}}public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate){Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate);}private string GetFile(string Category, string FileName, bool IsData){var path = dataPath + Category.NormalizeFileName() + @"\";if (!Directory.Exists(path))Directory.CreateDirectory(path);return path + FileName.NormalizeFileName() + "." + (IsData ? "dat" : ExpirationFileExtension);}private string GetKey(string Category, string Key){return Category + "_" + Key;}public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys){var result = new List<KeyValuePair<K, V>>();lock (syncRoot){Keys.ForEach(key =>{string buffer;V value;var cacheKey = GetKey(Category, key.ToString());if (isMemory){object memBuffer;if (memoryData.TryGetValue(cacheKey, out memBuffer)){//track recycleswitch (recycleAlgorithm){case RecycleAlgorithms.MRU:usageStat[cacheKey].Value++;break;default:break;}value = (V)memBuffer;DateTime? expirationDate;if (memoryExpiration.TryGetValue(cacheKey, out expirationDate)){//expiredif (expirationDate != null && (DateTime)expirationDate < DateTime.Now){value = default(V);memoryData.Remove(cacheKey);memoryExpiration.Remove(cacheKey);}}}elsevalue = default(V);}else{var dataFilePath = GetFile(Category, key.ToString(), true);if (File.Exists(dataFilePath)){buffer = File.ReadAllText(dataFilePath);//track recycleswitch (recycleAlgorithm){case RecycleAlgorithms.MRU:usageStat[cacheKey].Value++;break;default:break;}//will only deserialize object other than stringvalue = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer);DateTime expirationDate;var expirationFilePath = GetFile(Category, key.ToString(), false);if (File.Exists(expirationFilePath)){buffer = File.ReadAllText(expirationFilePath);expirationDate = Convert.ToDateTime(buffer);//expiredif (expirationDate < DateTime.Now){value = default(V);File.Delete(dataFilePath);File.Delete(expirationFilePath);}}}elsevalue = default(V);}result.Add(new KeyValuePair<K, V>(key, value));});}return result;}public override V Get<K, V>(string Category, K Key){var buffer = Get<K, V>(Category, new K[] { Key });var result = buffer.FirstOrDefault();return result.Value;}public override void Recycle<K>(string Category, long Count){if (Count < maxCount)return;switch (recycleAlgorithm){case RecycleAlgorithms.MRU:lock (syncRoot){var recycledFileCount = 0;if (isMemory){//find out expired itemsvar memExpired = memoryExpiration.Where(e => e.Value != null && (DateTime)e.Value < DateTime.Now);memExpired.ForEach(u =>{memoryData.Remove(u.Key);memoryExpiration.Remove(u.Key);usageStat.Remove(u.Key);});}else{if (expiredFiles.Count == 0){Directory.GetFiles(dataPath, "*." + ExpirationFileExtension).ForEach(f =>{var buffer = File.ReadAllText(f);var expirationDate = Convert.ToDateTime(buffer);expiredFiles[Path.GetFileNameWithoutExtension(f)] = new KeyValuePair<DateTime, string>(expirationDate, f);});}//find out expired itemsvar fileExpired = expiredFiles.Where(e => e.Value.Key < DateTime.Now);fileExpired.ForEach(u =>{var dataFile = Path.ChangeExtension(u.Value.Value, DataFileExtension);File.Delete(dataFile);File.Delete(u.Value.Value);usageStat.Remove(u.Key);recycledFileCount++;});}//find out least used itemsvar leastUsed = usageStat.OrderByDescending(s => s.Value.Value).Skip(maxCount - threshold);leastUsed.ForEach(u =>{if (isMemory){memoryData.Remove(u.Key);memoryExpiration.Remove(u.Key);}else{var dataFile = Path.ChangeExtension(u.Value.Key, DataFileExtension);if (File.Exists(dataFile)){recycledFileCount++;File.Delete(dataFile);}if (File.Exists(u.Value.Key))File.Delete(u.Value.Key);}usageStat.Remove(u.Key);});if (!isMemory){var statFilePath = dataPath + statFile;var count = 0;if (File.Exists(statFilePath)){var buffer = File.ReadAllText(statFilePath);count = Convert.ToInt32(buffer);}count = count - recycledFileCount;if (count < 0)count = 0;File.WriteAllText(statFilePath, count.ToString());}}break;default:break;}}}}

Extensions.cs

扩展函数

using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.IO;namespace Com.SuperCache.Engine{public static class Extensions{public static void ForEach<T>(this IEnumerable<T> source, Action<T> action){if (source != null){foreach (var item in source){action(item);}}}public static string NormalizeFileName(this string FileName){var result = FileName;Path.GetInvalidFileNameChars().ForEach(c =>{result = result.Replace(c.ToString(), string.Empty);});return result;}}}

新建

构造CacheEngine需要传递缓存保存到哪个文件夹。

基于内存

如果你不喜欢基于磁盘的缓存，可以使用基于内存，构造函数传递空字符串便可。

增加/更新

同一个方法：Add。用户可以指定类型（Category），譬如User，Employee等。键（Key）支持泛型，值（Data）是object。有一个overload是过期日期（ExpirationDate），默认当前时间30分钟后

获取

Get方法需要指定类型（Category）和键（Key）。

例子

using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Threading;using System.Threading.Tasks;using System.Diagnostics;using Com.SuperCache.Engine;namespace Com.SuperCache.Test{public class Foo{public string Name { get; set; }public int Age { get; set; }public double? Some { get; set; }public DateTime? Birthday { get; set; }}class Program{static void Main(string[] args){//TestAddGet();//Thread.Sleep(4000);//TestExpiration();TestDefaultDiskPerformance();TestDefaultMemoryPerformance();TestRawDiskPerformance();TestRawMemoryPerformance();//TestConcurrent();Console.Read();}private static void TestConcurrent(){var w = new Stopwatch();w.Start();Parallel.For(1, 3, (a) =>{var employees = Enumerable.Range((a - 1) * 1000, a * 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen"));var engine = new CacheEngine(@"..\..\data");engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1));});w.Stop();Console.WriteLine("add:" + w.Elapsed);var engine2 = new CacheEngine(@"..\..\data");var o = engine2.Get<int, string>("Employee", 1005);Console.WriteLine(o);}private static void TestDefaultDiskPerformance(){TestPerformance(CacheProviders.Default, @"..\..\data");}private static void TestDefaultMemoryPerformance(){TestPerformance(CacheProviders.Default, string.Empty);}private static void TestRawDiskPerformance(){TestPerformance(CacheProviders.Raw, @"..\..\data");}private static void TestRawMemoryPerformance(){TestPerformance(CacheProviders.Raw, string.Empty);}private static void TestPerformance(CacheProviders Provider, string DataPath){Console.WriteLine("Performance Test: " + Provider.ToString() + ", " + (string.IsNullOrEmpty(DataPath) ? "Memory" : DataPath));var engine = new CacheEngine(Provider, DataPath, RecycleAlgorithms.MRU, 900, 100, RecycleModes.Passive);var w = new Stopwatch();w.Start();var employees = Enumerable.Range(0, 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen"));engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1));w.Stop();Console.WriteLine("add:" + w.Elapsed);/*w.Restart();employees.ForEach(key =>{var o1 = engine.Get<int, string>("Employee", key.Key);});w.Stop();Console.WriteLine("individual get:" + w.Elapsed);*/w.Restart();var keys = employees.Select(i => i.Key);var o = engine.Get<int, string>("Employee", keys);w.Stop();Console.WriteLine("get:" + w.Elapsed);Console.WriteLine();}private static void TestExpiration(){var engine = new CacheEngine(@"..\..\data");var o = engine.Get<string, Foo>("User", "wchen");Console.WriteLine(o != null ? o.Name : "wchen does not exist or expired");}private static void TestAddGet(){var engine = new CacheEngine(@"..\..\data");var f = new Foo { Name = "Wilson Chen", Age = 30, Birthday = DateTime.Now, Some = 123.456 };engine.Add("User", "wchen", f, DateTime.Now.AddSeconds(5));var o = engine.Get<string, Foo>("User", "wchen");Console.WriteLine(o.Name);var o4 = engine.Get<string, Foo>("User", "foo");Console.WriteLine(o4 != null ? o4.Name : "foo does not exist");var o3 = engine.Get<string, string>("PlainText", "A");Console.WriteLine(o3 ?? "A does not exist");}}}

性能

通过上述性能测试例子，你会发现STSdb的磁盘存取速度要比一个记录对应一个物理文件快。想了解更多，请访问官方网站。

测试条件：1000条记录，7200RPM磁盘，i7。

说明

项目中引用了System.Management是因为STSdb支持内存数据库，需要判断最大物理内存。如果不喜欢，大家可以移除引用，并且去掉STSdb4.Database.STSdb.FromMemory方法便可。

下载

点击这里下载

如果觉得《C#开源磁盘/内存缓存引擎》对你有帮助，请点赞、收藏，并留下你的观点哦！