C# pythonnet(1)_感測器資料清洗演算法

Python程式碼如下

import pandas as pd

# 讀取資料
data = pd.read_csv('data_row.csv')

# 檢查異常值
def detect_outliers(data):
    outliers = []
    for col in data.columns:
        q1 = data[col].quantile(0.25)
        q3 = data[col].quantile(0.75)
        iqr = q3 - q1
        lower_bound = q1 - 1.5 * iqr
        upper_bound = q3 + 1.5 * iqr
        outliers.extend(data[(data[col] < lower_bound) | (data[col] > upper_bound)].index)
    return list(set(outliers))

outliers = detect_outliers(data)
print("異常資料數量：", len(outliers))
# 處理異常值
data.drop(outliers, inplace=True)

# 儲存清洗後的資料
data.to_csv('clean_data_row.csv', index=False)

下面我們修改成C#程式碼

建立控制檯程式，Nuget安裝 CsvHelper 和 pythonnet

public class Program
{
    const string PathToPythonDir = "D:\\Python311";
    const string DllOfPython = "python311.dll";

    static void Main(string[] args)
    {
        // 資料清洗
        CleanData();
    }

    /// <summary>
    /// 資料清洗
    /// </summary>
    static void CleanData()
    {
        var originDatas = ReadCsvWithCsvHelper("data_row.csv");

        var outliers = DetectOutliers(originDatas);

        var outlierHashset = new HashSet<int>(outliers);

        // 清洗過後的資料
        var cleanDatas = originDatas.Where((r, index) => !outlierHashset.Contains(index)).ToList();

        try
        {
            Runtime.PythonDLL = Path.Combine(PathToPythonDir, DllOfPython);

            PythonEngine.Initialize();
            using (Py.GIL())
            {
                dynamic pd = Py.Import("pandas");
                dynamic np = Py.Import("numpy");
                dynamic plt = Py.Import("matplotlib.pyplot");
                dynamic fft = Py.Import("scipy.fftpack");

                dynamic oData = np.array(originDatas.ToArray());
                int oDataLength = oData.__len__();
                dynamic data = np.array(cleanDatas.ToArray());
                int dataLength = data.__len__();

                // 繪製原始資料圖和清洗後資料圖
                plt.figure(figsize: new dynamic[] { 12, 6 });

                // 原始資料圖
                plt.subplot(1, 2, 1);
                plt.plot(np.arange(oDataLength), oData);
                plt.title("Original Datas");

                // 清洗後資料圖
                plt.subplot(1, 2, 2);
                plt.plot(np.arange(dataLength), data);
                plt.title("Clean Datas");

                // 佈局調整，防止重疊
                plt.tight_layout();
                // 顯示圖表
                plt.show();
            }
        }
        catch (Exception e)
        {
            Console.WriteLine("報錯了：" + e.Message + "\r\n" + e.StackTrace);
        }
    }

    /// <summary>
    /// 檢測異常值
    /// </summary>
    /// <param name="datas">原始資料集合</param>
    /// <returns>返回異常值在集合中的索引</returns>
    static List<int> DetectOutliers(List<double[]> datas)
    {
        List<int> outliers = new List<int>();
        var first = datas.First();
        for (int i = 0; i < first.Length; i++)
        {
            var values = datas.AsEnumerable().Select((row, index) => Tuple.Create(row[i], index)).ToArray();

            double q1 = Enumerable.OrderBy(values, x => x.Item1).ElementAt((int)(values.Length * 0.25)).Item1;
            double q3 = Enumerable.OrderBy(values, x => x.Item1).ElementAt((int)(values.Length * 0.75)).Item1;
            double iqr = q3 - q1;
            double lowerBound = q1 - 1.5 * iqr;
            double upperBound = q3 + 1.5 * iqr;

            outliers.AddRange(values.AsEnumerable()
                .Where(row => row.Item1 < lowerBound || row.Item1 > upperBound)
                .Select(row => row.Item2));
        }
        return outliers.Distinct().ToList();
    }


    /// <summary>
    /// 讀取CSV資料
    /// </summary>
    /// <param name="filePath">檔案路徑</param>
    /// <returns>檔案中資料集合，都是double型別</returns>
    static List<double[]> ReadCsvWithCsvHelper(string filePath)
    {
        using (var reader = new StreamReader(filePath))
        using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
        {
            var result = new List<double[]>();
            // 如果你的CSV檔案有標題行，可以呼叫ReadHeader來讀取它們
            csv.Read();
            csv.ReadHeader();
            while (csv.Read())
            {
                result.Add(new double[] {
                    csv.GetField<double>(0),
                    csv.GetField<double>(1),
                    csv.GetField<double>(2),
                });
            }
            return result;
        }
    }
}

以下是執行後結果，左邊是原始資料折線圖，右邊是清洗後資料折線圖

原始碼：https://gitee.com/Karl_Albright/csharp-demo/tree/master/PythonnetDemo/PythonnetClearData

抽稀演算法

def down_sampling(sig,factor=2, axis=0):
    '''
    降取樣
    Inputs:
        sig --- numpy array, 訊號資料陣列
        factor --- int, 降取樣倍率
        axis --- int, 沿著哪個軸進行降取樣
    '''
    Temp=[':']*sig.ndim
    Temp[axis]='::'+str(factor)
    return eval('sig['+','.join(Temp)+']')

/// <summary>
/// 降取樣，其實就是抽稀演算法
/// </summary>
static List<double[]> DownSampling(int factor = 2, int axis = 0)
{
    if (axis != 0 && axis != 1)
        throw new ArgumentException("Axis must be 0 or 1 for a 2D array.");

    var datas = ReadCsvWithCsvHelper("clean_data_row3.csv");

    int dim0 = datas.Count;
    var first = datas.First();
    int dim1 = first.Length;

    var result = new List<double[]>();
    if (axis == 0)
    {
        var xAxis = dim0 / factor;
        var yAxis = dim1;
        for (int i = 0; i < xAxis; i++)
        {
            result.Add(datas[i * factor]);
        }
    }
    else if (axis == 1)
    {
        var xAxis = dim0;
        var yAxis = dim1 / factor;
        var item = new double[yAxis];
        for (int i = 0; i < xAxis; i++)
        {
            var deviceData = datas[i];
            for (int j = 0; j < yAxis; j++)
            {
                item[j] = deviceData[j * factor];
            }
            result.Add(item);
        }
    }
    return result;
}

原始碼：https://gitee.com/Karl_Albright/csharp-demo/tree/master/PythonnetDemo/PythonnetClearData