稀疏映象在OpenHarmony上的應用
一、稀疏映象升級背景
常用系統映象格式為原始映象，即RAW格式。映象體積比較大，在燒錄韌體或者升級韌體時比較耗時，而且在移動裝置升級過程時比較耗費流量。為此，將原始映象用稀疏描述，可以大大地縮減映象體積，省時省流量。
二、稀疏映象原理
1、稀疏映象的概念
原始映象：即raw image，完整的ext4分割槽映象，包含很多全零的無效填充區
稀疏映象：即sparse image，將raw ext4進行稀疏描述，因此尺寸比較小，製作目錄有多少檔案就計算多少，沒有全零填充
2、稀疏映象格式
稀疏映象資料格式：首先是sparse_header佔用28byte，然後是12byte的chunk_header，同樣這chunk_header的型別決定了後面跟著的資料，如果讀到資料是0xCAC1意味著後面是本身的raw_data，如果是0xCAC3，則後面num為0，接著再0xCAC2意味著後面填充4byte的內容。
三、實現稀疏映象升級方案
版本基線：
OpenAtom OpenHarmony（以下簡稱“OpenHarmony”） 3.1 Release
程式碼路徑：
1、稀疏映象燒錄
（1）生成稀疏格式映象
有2種方法可以生成稀疏映象：
1）修改檔案build/ohos_var.gni中，sparse_image=true
2）編譯命令增加--sparse-image欄位，如./build.sh --product-name=xxx --sparse-image
（2）增加稀疏格式轉換工具
在目錄build/ohos/images/mkimage中增加檔案img2simg，該工具用於編譯完成後將raw映象轉換為sparse格式，並設定許可權為777。
（3）編譯後的映象對比
編譯出的映象格式為sparse格式，映象大小相比raw格式明顯變小。
（4）燒錄稀疏映象
燒錄稀疏映象方法和燒錄原始映象方法一致。
稀疏映象本身是不能直接掛載的，在燒錄過程中透過uboot將稀疏格式映象還原為原始映象，然後寫到磁碟中，系統啟動後可掛載對應的映象。
2、稀疏映象升級
升級包採用稀疏映象製作。
（1）修改升級包製作工具
官方升級包工具不支援生成稀疏映象的升級包，修改升級包工具，生成稀疏格式的升級包。.\base\update\packaging_tools\image_class.py
按照上圖所示註釋程式碼
（2）生成稀疏映象升級包
和全量映象升級包製作方法一致。
參考：
https://gitee.com/openharmony/docs/blob/master/zh-cn/device-dev/subsystems/subsys-ota-guide.md#%E6%A0%87%E5%87%86%E7%B3%BB%E7%BB%9F%E5%8D%87%E7%BA%A7%E5%8C%85%E5%88%B6%E4%BD%9C
（3）適配updater元件中稀疏映象功能
● 增加寫稀疏映象分支
.\base\update\updater\services\applypatch\data_writer.cpp
寫資料函式CreateDataWriter增加寫稀疏映象分支
case WRITE_SPARSE:
{
std::unique_ptr<SparseWriter> writer(std::make_unique<SparseWriter>(partitionName));
return std::move(writer);
}
● 增加稀疏映象類宣告
.\base\update\updater\services\applypatch\raw_writer.h
增加稀疏映象類宣告及相關變數定義
typedef struct sparse_header {
uint32_t magic; /* 0xed26ff3a */
uint16_t major_version; /* (0x1) - reject images with higher major versions */
uint16_t minor_version; /* (0x0) - allow images with higer minor versions */
uint16_t file_hdr_sz; /* 28 bytes for first revision of the file format */
uint16_t chunk_hdr_sz; /* 12 bytes for first revision of the file format */
uint32_t blk_sz; /* block size in bytes, must be a multiple of 4 (4096) */
uint32_t total_blks; /* total blocks in the non-sparse output image */
uint32_t total_chunks; /* total chunks in the sparse input image */
uint32_t image_checksum; /* CRC32 checksum of the original data, counting "don't care" */
/* as 0. Standard 802.3 polynomial, use a Public Domain */
/* table implementation */
} sparse_header_t;
#define SPARSE_HEADER_MAGIC 0xed26ff3a
#define CHUNK_TYPE_RAW 0xCAC1
#define CHUNK_TYPE_FILL 0xCAC2
#define CHUNK_TYPE_DONT_CARE 0xCAC3
#define CHUNK_TYPE_CRC32 0xCAC4
typedef struct chunk_header {
uint16_t chunk_type; /* 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care */
uint16_t reserved1;
uint32_t chunk_sz; /* in blocks in output image */
uint32_t total_sz; /* in bytes of chunk input file including chunk header and data */
} chunk_header_t;
class SparseWriter : public DataWriter {
public:
virtual bool Write(const uint8_t *addr, size_t len, WriteMode mode, const std::string &partitionName);
explicit SparseWriter(const std::string partitionName) : offset_(0), fd_(-1), partitionName_(partitionName) {}
virtual ~SparseWriter()
{
offset_ = 0;
if (fd_ > 0) {
fsync(fd_);
close(fd_);
}
fd_ = -1;
}
private:
int WriteInternal(int fd, const uint8_t *data, size_t len, const std::string &partitionName);
SparseWriter(const SparseWriter&) = delete;
const SparseWriter& operator=(const SparseWriter&) = delete;
off64_t offset_;
int fd_;
std::string partitionName_;
};
● 增加稀疏映象類實現
.\base\update\updater\services\applypatch\raw_writer.cpp
增加稀疏映象類實現及相關變數定義，原有程式碼不變
bool SparseWriter::Write(const uint8_t *addr, size_t len, WriteMode mode, const std::string &partitionName)
{
if (addr == nullptr) {
LOG(ERROR) << "SparseWriter: invalid address.";
return false;
}
if (len == 0) {
LOG(INFO) << "SparseWriter: write length is 0, skip.";
return false;
}
if (fd_ < 0) {
fd_ = OpenPartition(partitionName_);
if (fd_ < 0) {
return false;
}
}
UPDATER_CHECK_ONLY_RETURN(WriteInternal(fd_, addr, len, partitionName_) >= 0, return false);
return true;
}
int SparseWriter::WriteInternal(int fd, const uint8_t *data, size_t len, const std::string &partitionName)
{
uint32_t written = 0;
sparse_header_t *sparse_header;
chunk_header_t *chunk_header;
unsigned int chunk;
void *membuf = NULL;
uint32_t *fill_buf = NULL;
uint32_t fill_val;
uint32_t bytes_written = 0;
uint32_t total_bytes = 0;
uint32_t blk = 0;
uint32_t chunk_data_sz = 0;
uint32_t blkcnt = 0;
uint32_t blks = 0;
uint32_t total_blocks = 0;
uint32_t addr_offset = 0;
uint32_t fill_buf_num_blks = 0;
uint32_t block_size = 4096;
uint32_t block_count = 524288;
uint32_t i;
uint32_t j;
int ret = lseek64(fd, offset_, SEEK_SET);
UPDATER_FILE_CHECK(ret != -1, "RawWriter: failed to seek file to " << offset_, return -1);
fill_buf_num_blks = CONFIG_FASTBOOT_FLASH_FILLBUF_SIZE / block_size;
LOG(INFO) << "WriteInternal offset_ " << offset_;
/* Read and skip over sparse image header */
sparse_header = (sparse_header_t *)data;
data += sparse_header->file_hdr_sz;
if (sparse_header->file_hdr_sz > sizeof(sparse_header_t)) {
/*
* Skip the remaining bytes in a header that is longer than
* we expected.
*/
data += (sparse_header->file_hdr_sz - sizeof(sparse_header_t));
}
LOG(INFO) << "=== Sparse Image Header ===";
LOG(INFO) << "magic: " << sparse_header->magic;
LOG(INFO) << "major_version: " << sparse_header->major_version;
LOG(INFO) << "minor_version: " << sparse_header->minor_version;
LOG(INFO) << "file_hdr_sz: " << sparse_header->file_hdr_sz;
LOG(INFO) << "chunk_hdr_sz: " << sparse_header->chunk_hdr_sz;
LOG(INFO) << "blk_sz: " << sparse_header->blk_sz;
LOG(INFO) << "total_blks: " << sparse_header->total_blks;
LOG(INFO) << "total_chunks: " << sparse_header->total_chunks;
LOG(INFO) << "Flashing Sparse Image";
blk = 0;
for (chunk = 0; chunk < sparse_header->total_chunks; chunk++) {
/* Read and skip over chunk header */
chunk_header = (chunk_header_t *)data;
data += sizeof(chunk_header_t);
if (chunk_header->chunk_type != CHUNK_TYPE_RAW)
{
LOG(INFO) << "=== Chunk Header ===";
LOG(INFO) << "chunk_type: " << chunk_header->chunk_type;
LOG(INFO) << "chunk_sz: " << chunk_header->chunk_sz;
LOG(INFO) << "total_sz: " << chunk_header->total_sz;
}
if (sparse_header->chunk_hdr_sz > sizeof(chunk_header_t)) {
/*
* Skip the remaining bytes in a header that is longer
* than we expected.
*/
data += (sparse_header->chunk_hdr_sz -
sizeof(chunk_header_t));
}
chunk_data_sz = sparse_header->blk_sz * chunk_header->chunk_sz;
blkcnt = chunk_data_sz / block_size;
switch (chunk_header->chunk_type) {
case CHUNK_TYPE_RAW:
if (chunk_header->total_sz !=
(sparse_header->chunk_hdr_sz + chunk_data_sz)) {
LOG(ERROR) << "Bogus chunk size for chunk type Raw";
return -1;
}
if (blk + blkcnt > 0 + block_count) {
LOG(ERROR) << "Request would exceed partition size!";
return -1;
}
addr_offset = blk * block_size;
ret = lseek64(fd, offset_ + addr_offset, SEEK_SET);
if (ret < 0) {
LOG(ERROR) << "failed to seek file to " << addr_offset << " error=" << strerror(errno);
return -1;
}
written = write(fd, data, blkcnt * block_size);
if (written < 0) {
LOG(ERROR) << "SparseWriter: failed to write data of len ";
return -1;
}
total_bytes = total_bytes + blkcnt * block_size;
blks = written / block_size;
blk += blks;
bytes_written += blkcnt * block_size;
total_blocks += chunk_header->chunk_sz;
data += chunk_data_sz;
break;
case CHUNK_TYPE_FILL:
if (chunk_header->total_sz !=
(sparse_header->chunk_hdr_sz + sizeof(uint32_t))) {
LOG(ERROR) << "Bogus chunk size for chunk type FILL total_sz err " << chunk_header->total_sz << "\n";
return -1;
}
ret = posix_memalign (&membuf, 64,
ROUNDUP(
block_size * fill_buf_num_blks,
64));
if (ret) {
LOG(ERROR) << "posix_memalign:" << strerror (errno);
return -1;
}
fill_buf = (uint32_t *)membuf;
if (!fill_buf) {
LOG(ERROR) << "Malloc failed for: CHUNK_TYPE_FILL";
return -1;
}
fill_val = *(uint32_t *)data;
data = data + sizeof(uint32_t);
for (i = 0;
i < (block_size * fill_buf_num_blks /
sizeof(fill_val));
i++)
fill_buf[i] = fill_val;
if (blk + blkcnt > 0 + block_count) {
LOG(ERROR) << "Request would exceed partition size!";
return -1;
}
for (i = 0; i < blkcnt;) {
j = blkcnt - i;
if (j > fill_buf_num_blks)
j = fill_buf_num_blks;
addr_offset = blk * block_size;
ret = lseek64(fd, offset_ + addr_offset, SEEK_SET);
if (ret < 0) {
LOG(ERROR) << "failed to lseek file to " << addr_offset << " error=" << strerror(errno);
return -1;
}
written = write(fd, fill_buf, j * block_size);
if (written < 0) {
LOG(ERROR) << "SparseWriter: failed to write data of len ";
return -1;
}
total_bytes = total_bytes + j * block_size;
blks = written / block_size;
if (blks < j) {
LOG(ERROR) << "Write failed, block";
free(fill_buf);
return -1;
}
blk += blks;
i += j;
}
bytes_written += blkcnt * block_size;
total_blocks += chunk_data_sz / sparse_header->blk_sz;
free(fill_buf);
break;
case CHUNK_TYPE_DONT_CARE:
blk += blkcnt;
total_blocks += chunk_header->chunk_sz;
break;
case CHUNK_TYPE_CRC32:
if (chunk_header->total_sz !=
sparse_header->chunk_hdr_sz) {
LOG(ERROR) << "Bogus chunk size for chunk type CRC32 total_sz err " << chunk_header->total_sz;
return -1;
}
total_blocks += chunk_header->chunk_sz;
data += chunk_data_sz;
break;
default:
LOG(INFO) << __func__ << ": Unknown chunk type: " << chunk_header->chunk_type;
return -1;
}
}
LOG(INFO) << "Wrote "<< chunk <<"blocks, expected to write " << sparse_header->total_blks << "blocks\n";
LOG(INFO) << "........ wrote "<< bytes_written <<"bytes to " << partitionName << "\n";
LOG(INFO) << "total_bytes=" << total_bytes;
return 0;
}
3、驗證稀疏映象升級
（1）複製升級包
將製作的稀疏映象升級包透過HDC工具推進系統data目錄，並修改檔名為updater.zip，路徑如下：/data/updater/updater.zip
（2）啟動升級
系統啟動後，在命令列工具輸入如下命令啟動升級
reboot updater:--update_package=/data/updater/updater.zip
輸入命令後，系統重啟，進入升級頁面，等待升級完成。
本文介紹了OpenHarmony系統中實現稀疏映象升級的方法，理解稀疏映象原理及稀疏映象還原方法可以快速在自己的系統中應用稀疏映象升級，提高系統升級速度。
來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/70011554/viewspace-2934391/，如需轉載，請註明出處，否則將追究法律責任。