leveldb.net物件讀寫封裝

smark發表於2014-09-03
物件封裝

        leveldb是一個非常高效的可嵌入式K-V資料庫,在.NET下有著基於win實現的包裝leveldb.net;不過leveldb.net只提供了基於byte[]和string的處理,這顯然會對使用的時候帶來不方便,畢竟在編寫應用的時候都是希望通過物件的方式來儲存,如我們常見的redis,mongodb和memcached等等都提供物件方式的讀寫.以下主要講解leveldb.net基礎上封裝一層序列化功能方便使用.

制定物件化的訪問介面

為了不修改leveldb.net的程式碼,所以選擇在他基礎過行封裝,為了清楚需要些什麼簡單地定義了一個規則

 public interface IDBManager
    {
        IFormater Formater { get; set; }

        void Set(string key, object data);

        object Get(string key, Type type);

        T Get<T>(string key);

        void Open();

        LevelDB.DB DataBase
        {
            get;
        }
       
    }

程式碼非常簡單主要封裝了GET,SET,實際上還有DELETE操作,這裡偷懶就沒做了:),為了提供靈活序列化規則所以在這個管理介面上還提供了一個Formater屬性.下面是這相介面的詳細實現:

 

public class LevelDBManager : IDBManager
    {

        public LevelDBManager()
        {
            
        }

        private LevelDB.DB mDataBase;

        public string Path { get; set; }

        public IFormater Formater
        {
            get;
            set;
        }

        public void Open()
        {
            mDataBase = new LevelDB.DB(Path, new Options() { CreateIfMissing = true });
          
        }

        public void Set(string key, object data)
        {

            FormaterBuffer buffer = Formater.Pop();
            try
            {

                int count = Formater.Serialize(data, buffer, 0);
                mDataBase.Put(Encoding.UTF8.GetBytes(key), buffer.Array, 0, count);
            }
            finally
            {
                Formater.Push(buffer);
            }
        }

        public object Get(string key, Type type)
        {
            FormaterBuffer buffer = Formater.Pop();
            long count;
            object result = null;
           
            try
            {
                count = mDataBase.Get(Encoding.UTF8.GetBytes(key), buffer.Array);
                if (count > 0)
                {
                    result = Formater.Deserialize(type, buffer, 0, (int)count);

                }
                return result;
            }
            finally
            {
                Formater.Push(buffer);
            }
          
        }

        public T Get<T>(string key)
        {
            return (T)Get(key, typeof(T));
        }


        public DB DataBase
        {
            get { return mDataBase; }
        }
    }

相信以上那些簡知的程式碼也比較好理解,所以就不詳細說明了.

可擴充套件的序列化規則

由於在使用上的需要,都習慣用些不同序列化方式來進行物件序列化,這個封裝為了實現一個比較高的靈活度,所以物件序列化過程也制定了一個介面進行隔離.主要為了滿足不同人的胃口.

 

   public interface IFormater
    {
        FormaterBuffer Pop();

        void Push(FormaterBuffer data);

        int Serialize(object data, FormaterBuffer buffer, int offset);

        object Deserialize(Type type, FormaterBuffer buffer, int offset, int count);
    }

比較簡單定義了序列化和反序列化的方法,不過為了一些效能上的考慮增加了buffer的複用功能,這個設計緊緊用作需要追求這方面效能要求而準備.下面看一下json和protobuf的實現是怎樣的:

 public abstract class FormaterBase:IFormater
    {
         private Stack<FormaterBuffer> mBufferPool = new Stack<FormaterBuffer>();

        const int BUFFER_SIZE = 1024 * 1024 * 1;

        public FormaterBase()
        {
            for (int i = 0; i < 20; i++)
            {
                mBufferPool.Push(new FormaterBuffer(BUFFER_SIZE));
            }
        }
        public FormaterBuffer Pop()
        {
            lock (mBufferPool)
            {
                if(mBufferPool.Count>0)
                    return mBufferPool.Pop();
                return new FormaterBuffer(BUFFER_SIZE);
            }
        }
        public void Push(FormaterBuffer data)
        {
            lock (mBufferPool)
            {
                mBufferPool.Push(data);
            }
        }
       
        public abstract int Serialize(object data, FormaterBuffer buffer, int offset);
       
        public abstract object Deserialize(Type type, FormaterBuffer buffer, int offset, int count);
        
    }
  • json 
    public class JsnoFormater:FormaterBase
    {
       
        public int Serialize(object data, byte[] buffer, int offset)
        {
            string json = Newtonsoft.Json.JsonConvert.SerializeObject(data);
            return Encoding.UTF8.GetBytes(json, 0, json.Length, buffer, offset);
        }

        public override int Serialize(object data, FormaterBuffer buffer, int offset)
        {
            string json = Newtonsoft.Json.JsonConvert.SerializeObject(data);
            return Encoding.UTF8.GetBytes(json, 0, json.Length, buffer.Array, offset);
        }

        public override object Deserialize(Type type, FormaterBuffer buffer, int offset, int count)
        {
            string value = Encoding.UTF8.GetString(buffer.Array, offset, count);
            return Newtonsoft.Json.JsonConvert.DeserializeObject(value, type);
        }
    }
  • protobuf 
     public class ProtobufFormater:FormaterBase
    {

        public override int Serialize(object data, FormaterBuffer buffer, int offset)
        {
            buffer.Seek(offset);
            ProtoBuf.Meta.RuntimeTypeModel.Default.Serialize(buffer.Stream, data);
            return (int)buffer.Stream.Position;
        }

        public override object Deserialize(Type type, FormaterBuffer buffer, int offset, int count)
        {
            buffer.Stream.SetLength(count + offset);
            buffer.Seek(offset);
            return ProtoBuf.Meta.RuntimeTypeModel.Default.Deserialize(buffer.Stream, null, type);
        }
    }

leveldb.net的一些簡單效能改造

雖然leveldb.net只以win dll的基礎上包裝,但在包裝過程的確有些方法針對我個人來說做得並不理想,主要體現在buffer複用方面.其實get,set方法都存在這情況.

 

 /// <summary>
        /// Set the database entry for "key" to "value".  
        /// </summary>
        public void Put(byte[] key, byte[] value, WriteOptions options)
        {
            IntPtr error;
            LevelDBInterop.leveldb_put(this.Handle, options.Handle, key, (IntPtr)key.Length, value, (IntPtr)value.LongLength, out error);
            LevelDBException.Check(error);
            GC.KeepAlive(options);
            GC.KeepAlive(this);
        }
        public unsafe byte[] Get(byte[] key, ReadOptions options)
        {
            IntPtr error;
            IntPtr lengthPtr;
            var valuePtr = LevelDBInterop.leveldb_get(this.Handle, options.Handle, key, (IntPtr)key.Length, out lengthPtr, out error);
            LevelDBException.Check(error);
            if (valuePtr == IntPtr.Zero)
                return null;
            try
            {
                var length = (long)lengthPtr;
                var value = new byte[length];
                var valueNative = (byte*)valuePtr.ToPointer();
                for (long i = 0; i < length; ++i)
                    value[i] = valueNative[i];
                return value;
            }
            finally
            {
                LevelDBInterop.leveldb_free(valuePtr);
                GC.KeepAlive(options);
                GC.KeepAlive(this);
            }
        }

兩上個方法都不支援從外部帶入buffer的情況,當需要高併發操作的情況而物件序列化內容又比較大的情況下,那的確是會讓人感覺到不滿意.所以在這基礎上新增了一些有利於buffer複用的方法來支援高併發操作下的效能需要.

 

 public void Put(byte[] key, byte[] value, int offset, int length, WriteOptions options)
        {
            IntPtr error;
            LevelDBInterop.leveldb_put(this.Handle, options.Handle, key, (IntPtr)key.Length, value, (IntPtr)length, out error);
            LevelDBException.Check(error);
            GC.KeepAlive(options);
            GC.KeepAlive(this);
        }
        public unsafe long Get(byte[] key, byte[] buffer, ReadOptions options)
        {
            IntPtr error;
            IntPtr lengthPtr;
            var valuePtr = LevelDBInterop.leveldb_get(this.Handle, options.Handle, key, (IntPtr)key.Length, out lengthPtr, out error);
            LevelDBException.Check(error);
            if (valuePtr == IntPtr.Zero)
                return 0;
            try
            {
                var length = (long)lengthPtr;
                var valueNative = (byte*)valuePtr.ToPointer();
                Marshal.Copy((IntPtr)valuePtr, buffer, 0, (int)length);
                return length;
            }
            finally
            {
                LevelDBInterop.leveldb_free(valuePtr);
              
            }
        }

相關文章