From 553b0ec49bc64fc4b7df4358cd31396a87276d2b Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Sat, 26 Jan 2013 01:00:39 -0800 Subject: Update payload library + command-line tool An initial implementation of a Python module for parsing, checking and applying a Chrome OS update payload. Comes with a command-line tool (paycheck.py) for applying such operations on payload files, and a test script (test_paycheck.sh) for ensuring that the library and tool are working correctly. Since update_payload is introduced as a package, we're moving some previously merged utilities into the package's directory. (Unit testing for this code will be uploaded on a separate CL; see chromium-os:39663) BUG=chromium-os:34911,chromium-os:33607,chromium-os:7597 TEST=test_paycheck.sh successful on MP-signed payloads CQ-DEPEND=I5746a1d80e822a575f0d96f94d0b4e765fc64507 Change-Id: I77123a1fffbb2059c239b7145c6922968fdffb6a Reviewed-on: https://gerrit.chromium.org/gerrit/43041 Reviewed-by: Gilad Arnold Tested-by: Gilad Arnold Reviewed-by: Chris Sosa Reviewed-by: Jay Srinivasan Reviewed-by: Don Garrett Commit-Queue: Gilad Arnold --- scripts/update_payload/payload.py | 257 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 scripts/update_payload/payload.py (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py new file mode 100644 index 00000000..6dda644a --- /dev/null +++ b/scripts/update_payload/payload.py @@ -0,0 +1,257 @@ +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tools for reading, verifying and applying Chrome OS update payloads.""" + +import hashlib +import struct + +import applier +import block_tracer +import checker +import common +from error import PayloadError +import update_metadata_pb2 + + +# +# Helper functions. +# +def _ReadInt(file_obj, size, is_unsigned, hasher=None): + """Read a binary-encoded integer from a file. + + It will do the correct conversion based on the reported size and whether or + not a signed number is expected. Assumes a network (big-endian) byte + ordering. + + Args: + file_obj: a file object + size: the integer size in bytes (2, 4 or 8) + is_unsigned: whether it is signed or not + hasher: an optional hasher to pass the value through + Returns: + An "unpacked" (Python) integer value. + Raises: + PayloadError if an read error occurred. + + """ + # Determine the base conversion format. + if size == 2: + fmt = 'h' + elif size == 4: + fmt = 'i' + elif size == 8: + fmt = 'q' + else: + raise PayloadError('unsupport numeric field size (%s)' % size) + + # Signed or unsigned? + if is_unsigned: + fmt = fmt.upper() + + # Our numeric values are in network byte order (big-endian). + fmt = '!' + fmt + + return struct.unpack(fmt, common.Read(file_obj, size, hasher=hasher))[0] + + +# +# Update payload. +# +class Payload(object): + """Chrome OS update payload processor.""" + + class _PayloadHeader(object): + """Update payload header struct.""" + + def __init__(self, version, manifest_len): + self.version = version + self.manifest_len = manifest_len + + # Header constants; sizes are in bytes. + _MAGIC = 'CrAU' + _VERSION_SIZE = 8 + _MANIFEST_LEN_SIZE = 8 + + def __init__(self, payload_file): + """Initialize the payload object. + + Args: + payload_file: update payload file object open for reading + + """ + self.payload_file = payload_file + self.manifest_hasher = None + self.is_init = False + self.header = None + self.manifest = None + self.data_offset = 0 + + def _ReadHeader(self): + """Reads and returns the payload header. + + Returns: + A payload header object. + Raises: + PayloadError if a read error occurred. + + """ + # Verify magic + magic = common.Read(self.payload_file, len(self._MAGIC), + hasher=self.manifest_hasher) + if magic != self._MAGIC: + raise PayloadError('invalid payload magic: %s' % magic) + + return self._PayloadHeader( + _ReadInt(self.payload_file, self._VERSION_SIZE, True, + hasher=self.manifest_hasher), + _ReadInt(self.payload_file, self._MANIFEST_LEN_SIZE, True, + hasher=self.manifest_hasher)) + + def _ReadManifest(self): + """Reads and returns the payload manifest. + + Returns: + A string containing the payload manifest in binary form. + Raises: + PayloadError if a read error occurred. + + """ + if not self.header: + raise PayloadError('payload header not present') + + return common.Read(self.payload_file, self.header.manifest_len, + hasher=self.manifest_hasher) + + def ReadDataBlob(self, offset, length): + """Reads and returns a single data blob from the update payload. + + Args: + offset: offset to the beginning of the blob from the end of the manifest + length: the blob's length + Returns: + A string containing the raw blob data. + Raises: + PayloadError if a read error occurred. + + """ + return common.Read(self.payload_file, length, + offset=self.data_offset + offset) + + def Init(self): + """Initializes the payload object. + + This is a prerequisite for any other public API call. + + Raises: + PayloadError if object already initialized or fails to initialize + correctly. + + """ + if self.is_init: + raise PayloadError('payload object already initialized') + + # Initialize hash context. + # pylint: disable=E1101 + self.manifest_hasher = hashlib.sha256() + + # Read the file header. + self.header = self._ReadHeader() + + # Read the manifest. + manifest_raw = self._ReadManifest() + self.manifest = update_metadata_pb2.DeltaArchiveManifest() + self.manifest.ParseFromString(manifest_raw) + + # Store data offset. + self.data_offset = (len(self._MAGIC) + self._VERSION_SIZE + + self._MANIFEST_LEN_SIZE + self.header.manifest_len) + + self.is_init = True + + def _AssertInit(self): + """Raises an exception if the object was not initialized.""" + if not self.is_init: + raise PayloadError('payload object not initialized') + + def ResetFile(self): + """Resets the offset of the payload file to right past the manifest.""" + self.payload_file.seek(self.data_offset) + + def IsDelta(self): + """Returns True iff the payload appears to be a delta.""" + self._AssertInit() + return (self.manifest.HasField('old_kernel_info') or + self.manifest.HasField('old_rootfs_info')) + + def IsFull(self): + """Returns True iff the payload appears to be a full.""" + return not self.IsDelta() + + def Check(self, pubkey_file_name=None, metadata_sig_file=None, + report_out_file=None, assert_type=None, block_size=0, + allow_unhashed=False): + """Checks the payload integrity. + + Args: + pubkey_file_name: public key used for signature verification + metadata_sig_file: metadata signature, if verification is desired + report_out_file: file object to dump the report to + assert_type: assert that payload is either 'full' or 'delta' + block_size: expected filesystem / payload block size + allow_unhashed: allow unhashed operation blobs + Raises: + PayloadError if payload verification failed. + + """ + self._AssertInit() + + # Create a short-lived payload checker object and run it. + helper = checker.PayloadChecker(self) + helper.Run(pubkey_file_name=pubkey_file_name, + metadata_sig_file=metadata_sig_file, + report_out_file=report_out_file, assert_type=assert_type, + block_size=block_size, allow_unhashed=allow_unhashed) + + def Apply(self, dst_kernel_part, dst_rootfs_part, src_kernel_part=None, + src_rootfs_part=None): + """Applies the update payload. + + Args: + dst_kernel_part: name of dest kernel partition file + dst_rootfs_part: name of dest rootfs partition file + src_kernel_part: name of source kernel partition file (optional) + src_rootfs_part: name of source rootfs partition file (optional) + Raises: + PayloadError if payload application failed. + + """ + self._AssertInit() + + # Create a short-lived payload applier object and run it. + helper = applier.PayloadApplier(self) + helper.Run(dst_kernel_part, dst_rootfs_part, + src_kernel_part=src_kernel_part, + src_rootfs_part=src_rootfs_part) + + def TraceBlock(self, block, skip, trace_out_file, is_kernel): + """Traces the origin(s) of a given dest partition block. + + The tracing tries to find origins transitively, when possible (it currently + only works for move operations, where the mapping of src/dst is + one-to-one). It will dump a list of operations and source blocks + responsible for the data in the given dest block. + + Args: + block: the block number whose origin to trace + skip: the number of first origin mappings to skip + trace_out_file: file object to dump the trace to + is_kernel: trace through kernel (True) or rootfs (False) operations + + """ + self._AssertInit() + + # Create a short-lived payload block tracer object and run it. + helper = block_tracer.PayloadBlockTracer(self) + helper.Run(block, skip, trace_out_file, is_kernel) -- cgit v1.2.3 From 5502b56f34f9703cf053be46e4ea5685c0c9ac26 Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Fri, 8 Mar 2013 13:22:31 -0800 Subject: paycheck: unit tests + fixes to checker module This adds missing unit tests for the checker module, bundled with fixes to some bugs that surfaced due to unit tests. This includes: * A fake extent (signified by start_block == UINT64_MAX) that accompanies a signature data blob bears different requirements than previously implemented. Specifically, the extent sequence must have exactly one extent; and the number of blocks is not necessarily one, rather it is the correct number that corresponds to the actual length of the signature blob. * REPLACE/REPLACE_BZ operations must contain data. * MOVE operation validation must ensure that all of the actual message extents are being used. * BSDIFF operation must contain data (the diff). * Signature pseudo-operation should be a REPLACE. BUG=chromium-os:34911,chromium-os:33607,chromium-os:7597 TEST=Passes unittests (upcoming); works with actual payloads. Change-Id: I4d839d1d4da1fbb4a493b208958a139368e2c8ca Reviewed-on: https://gerrit.chromium.org/gerrit/45429 Tested-by: Gilad Arnold Reviewed-by: Chris Sosa Commit-Queue: Gilad Arnold --- scripts/update_payload/payload.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index 6dda644a..dbb385a1 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -19,7 +19,7 @@ import update_metadata_pb2 # Helper functions. # def _ReadInt(file_obj, size, is_unsigned, hasher=None): - """Read a binary-encoded integer from a file. + """Reads a binary-encoded integer from a file. It will do the correct conversion based on the reported size and whether or not a signed number is expected. Assumes a network (big-endian) byte @@ -36,24 +36,8 @@ def _ReadInt(file_obj, size, is_unsigned, hasher=None): PayloadError if an read error occurred. """ - # Determine the base conversion format. - if size == 2: - fmt = 'h' - elif size == 4: - fmt = 'i' - elif size == 8: - fmt = 'q' - else: - raise PayloadError('unsupport numeric field size (%s)' % size) - - # Signed or unsigned? - if is_unsigned: - fmt = fmt.upper() - - # Our numeric values are in network byte order (big-endian). - fmt = '!' + fmt - - return struct.unpack(fmt, common.Read(file_obj, size, hasher=hasher))[0] + return struct.unpack(common.IntPackingFmtStr(size, is_unsigned), + common.Read(file_obj, size, hasher=hasher))[0] # -- cgit v1.2.3 From eaed0d1371d781d3f5effa1475f5202dea9467e7 Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Tue, 30 Apr 2013 15:38:22 -0700 Subject: paycheck: allow to disable specific checks This became necessary as the delta generator appears to generate payloads that fail certain checks (e.g. during update_engine unit testing). BUG=None TEST=Disabled checks not being triggered Change-Id: I4491e0cb32ef44f85e11ffb0402b40d1371525ae Reviewed-on: https://gerrit.chromium.org/gerrit/49676 Tested-by: Gilad Arnold Reviewed-by: Chris Sosa Commit-Queue: Gilad Arnold --- scripts/update_payload/payload.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index dbb385a1..b4760b26 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -175,7 +175,7 @@ class Payload(object): def Check(self, pubkey_file_name=None, metadata_sig_file=None, report_out_file=None, assert_type=None, block_size=0, - allow_unhashed=False): + allow_unhashed=False, disabled_tests=()): """Checks the payload integrity. Args: @@ -185,6 +185,7 @@ class Payload(object): assert_type: assert that payload is either 'full' or 'delta' block_size: expected filesystem / payload block size allow_unhashed: allow unhashed operation blobs + disabled_tests: list of tests to disable Raises: PayloadError if payload verification failed. @@ -192,11 +193,12 @@ class Payload(object): self._AssertInit() # Create a short-lived payload checker object and run it. - helper = checker.PayloadChecker(self) + helper = checker.PayloadChecker( + self, assert_type=assert_type, block_size=block_size, + allow_unhashed=allow_unhashed, disabled_tests=disabled_tests) helper.Run(pubkey_file_name=pubkey_file_name, metadata_sig_file=metadata_sig_file, - report_out_file=report_out_file, assert_type=assert_type, - block_size=block_size, allow_unhashed=allow_unhashed) + report_out_file=report_out_file) def Apply(self, dst_kernel_part, dst_rootfs_part, src_kernel_part=None, src_rootfs_part=None): -- cgit v1.2.3 From 382df5ce2f4b67bf0998b01c6fedcdb5c35ebef9 Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Fri, 3 May 2013 12:49:28 -0700 Subject: paycheck: enforce physical partition size correctly During payload checking, payload has wrongly interpreted the size reported in the update payload to be the physical partition size, whereas this is in fact the size of the filesystem portion only (a misnomer). This sometimes caused it to emit errors on out-of-bounds operations, which are otherwise harmless in real-world scenarios. This CL makes a clear distinction between the two, with the following semantics: - The payload's embedded filesystem size must by <= the physical partition sizes. - Reading/writing from/to the new partition must be within the physical partition size boundaries, and not the filesystem ones. - Reading from the old partition is only allowed from filesystem boundaries; this is unchanged from current behavior and appears to be consistent with how we perform delta updates. - Old/new SHA256 verification during payload application is now limited to the allotted filesystem portion only (and not the full partition size). This is consistent with the update engine's semantics. - Other than that, this change currently has no further effect on payload application, which remains more permissive wrt to partition sizes. This also means that the sizes of partitions resulting from a payload application will not necessarily abide by the predetermined physical partition sizes. This is in line with the prevailing division of responsibilities between payload checking (strict) and application (relaxed). BUG=chromium:221847 TEST=Payload checking respects partition size override TEST=Unit tests pass TEST=Integration tests pass Change-Id: I0dbc88d538c0cc53b7551f4dfa8f543bcf480cd5 Reviewed-on: https://gerrit.chromium.org/gerrit/50103 Reviewed-by: Gilad Arnold Tested-by: Gilad Arnold Commit-Queue: David James --- scripts/update_payload/payload.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index b4760b26..e4320929 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -175,7 +175,8 @@ class Payload(object): def Check(self, pubkey_file_name=None, metadata_sig_file=None, report_out_file=None, assert_type=None, block_size=0, - allow_unhashed=False, disabled_tests=()): + rootfs_part_size=0, kernel_part_size=0, allow_unhashed=False, + disabled_tests=()): """Checks the payload integrity. Args: @@ -184,6 +185,8 @@ class Payload(object): report_out_file: file object to dump the report to assert_type: assert that payload is either 'full' or 'delta' block_size: expected filesystem / payload block size + rootfs_part_size: the size of (physical) rootfs partitions in bytes + kernel_part_size: the size of (physical) kernel partitions in bytes allow_unhashed: allow unhashed operation blobs disabled_tests: list of tests to disable Raises: @@ -198,6 +201,8 @@ class Payload(object): allow_unhashed=allow_unhashed, disabled_tests=disabled_tests) helper.Run(pubkey_file_name=pubkey_file_name, metadata_sig_file=metadata_sig_file, + rootfs_part_size=rootfs_part_size, + kernel_part_size=kernel_part_size, report_out_file=report_out_file) def Apply(self, dst_kernel_part, dst_rootfs_part, src_kernel_part=None, -- cgit v1.2.3 From 16416600a92a60294cd57aceec170a13ed72ed19 Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Sat, 4 May 2013 21:40:39 -0700 Subject: paycheck: src/dst -> old/new in applier code The old/new terminology is consistent with that used in update payload manifest, as well as other parts of the update_payload library (e.g. checker). BUG=None TEST=Unit/integration test passes Change-Id: I91244ae8adf073b958e8cd7e7670341f056c848e Reviewed-on: https://gerrit.chromium.org/gerrit/50130 Tested-by: Gilad Arnold Reviewed-by: Don Garrett Commit-Queue: Gilad Arnold --- scripts/update_payload/payload.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index e4320929..34ebaae7 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -205,15 +205,15 @@ class Payload(object): kernel_part_size=kernel_part_size, report_out_file=report_out_file) - def Apply(self, dst_kernel_part, dst_rootfs_part, src_kernel_part=None, - src_rootfs_part=None): + def Apply(self, new_kernel_part, new_rootfs_part, old_kernel_part=None, + old_rootfs_part=None): """Applies the update payload. Args: - dst_kernel_part: name of dest kernel partition file - dst_rootfs_part: name of dest rootfs partition file - src_kernel_part: name of source kernel partition file (optional) - src_rootfs_part: name of source rootfs partition file (optional) + new_kernel_part: name of dest kernel partition file + new_rootfs_part: name of dest rootfs partition file + old_kernel_part: name of source kernel partition file (optional) + old_rootfs_part: name of source rootfs partition file (optional) Raises: PayloadError if payload application failed. @@ -222,9 +222,9 @@ class Payload(object): # Create a short-lived payload applier object and run it. helper = applier.PayloadApplier(self) - helper.Run(dst_kernel_part, dst_rootfs_part, - src_kernel_part=src_kernel_part, - src_rootfs_part=src_rootfs_part) + helper.Run(new_kernel_part, new_rootfs_part, + old_kernel_part=old_kernel_part, + old_rootfs_part=old_rootfs_part) def TraceBlock(self, block, skip, trace_out_file, is_kernel): """Traces the origin(s) of a given dest partition block. -- cgit v1.2.3 From 272a499e2db9d72a64490ca5ccbebe8155fc2966 Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Wed, 8 May 2013 13:12:53 -0700 Subject: paycheck: support for in-place BSDIFF operations When applying BSDIFF operations in payloads, we used to extract the block sequences from the old partition and serialize it into a temporary file, and vice versa for the new partition. This worked but did not allow us to test bspatch as it is actually being used by the update engine. This CLs allows paycheck to invoke bspatch so that it reads/writes block extents directly from/to the partition file, in the same way it's being used by the update engine. Since performance is the same, this is the new default behavior; users can opt for the aforementioned old behavior using a command-line flag (-x or --extract-bsdiff). BUG=chromium:229705 TEST=bspatch invoked differently depending on the -x flag TEST=Passes unit/integration tests Change-Id: I8821754e1163b357617ece6befa42d1c2e575930 Reviewed-on: https://gerrit.chromium.org/gerrit/50486 Tested-by: Gilad Arnold Reviewed-by: Darin Petkov Commit-Queue: Gilad Arnold --- scripts/update_payload/payload.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index 34ebaae7..fefc7e4f 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -206,7 +206,7 @@ class Payload(object): report_out_file=report_out_file) def Apply(self, new_kernel_part, new_rootfs_part, old_kernel_part=None, - old_rootfs_part=None): + old_rootfs_part=None, bsdiff_in_place=True): """Applies the update payload. Args: @@ -214,6 +214,7 @@ class Payload(object): new_rootfs_part: name of dest rootfs partition file old_kernel_part: name of source kernel partition file (optional) old_rootfs_part: name of source rootfs partition file (optional) + bsdiff_in_place: whether to perform BSDIFF operations in-place (optional) Raises: PayloadError if payload application failed. @@ -221,7 +222,7 @@ class Payload(object): self._AssertInit() # Create a short-lived payload applier object and run it. - helper = applier.PayloadApplier(self) + helper = applier.PayloadApplier(self, bsdiff_in_place=bsdiff_in_place) helper.Run(new_kernel_part, new_rootfs_part, old_kernel_part=old_kernel_part, old_rootfs_part=old_rootfs_part) -- cgit v1.2.3 From 432d601e236bf8b9110fdb497e5f5c87899346e2 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 10 May 2013 15:01:36 -0700 Subject: Update paycheck to understand the updated update_metadata fields. When checking a payload, always start by printing a description of the payload being checked. Recompiled (with protoc) the updated update_metadata.proto from the update_engine. BUG=chromium:226310 TEST=Manual Run CQ-DEPEND=CL:47347 Change-Id: Ib60c6e6978d30444db7b65ef6d09540c9ffacdb8 Reviewed-on: https://gerrit.chromium.org/gerrit/50899 Commit-Queue: Don Garrett Reviewed-by: Don Garrett Tested-by: Don Garrett --- scripts/update_payload/payload.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index fefc7e4f..1796f511 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -154,6 +154,30 @@ class Payload(object): self.is_init = True + def Describe(self): + + def _DescribeImageInfo(description, image_info): + def _DisplayIndentedValue(name, value): + print ' {:<14} {}'.format(name+':', value) + + print '%s:' % description + _DisplayIndentedValue('Channel', image_info.channel) + _DisplayIndentedValue('Board', image_info.board) + _DisplayIndentedValue('Version', image_info.version) + _DisplayIndentedValue('Key', image_info.key) + + if (image_info.build_channel != image_info.channel): + _DisplayIndentedValue('Build channel', image_info.build_channel) + + if (image_info.build_version != image_info.version): + _DisplayIndentedValue('Build version', image_info.build_version) + + if self.manifest.HasField('old_image_info'): + _DescribeImageInfo('Old Image', self.manifest.old_image_info) + + if self.manifest.HasField('new_image_info'): + _DescribeImageInfo('New Image', self.manifest.new_image_info) + def _AssertInit(self): """Raises an exception if the object was not initialized.""" if not self.is_init: -- cgit v1.2.3 From e5fdf189ce3a4628f02a0bd5e09694bf7b815cdf Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Thu, 23 May 2013 16:13:38 -0700 Subject: paycheck: truncate partitions resulting from applying a payload Since the correctness of the result only encompasses the filesystem (or otherwise "meaningful data") on the target partition, it is desirable to actually get rid of whatever is past that point. There are different reasons for the presence of such extra space in delta updates, including remnants from a source partition that served as baseline for a delta update, or scratch space used by MOVE operations for breaking cycles. We make truncation the default behavior, although it can be suppressed by passing the right flag (truncate_to_expected_size=False). Note that this change is necessary for comparing the results of applying a payload to the partitions as they are extracted from a target image, which is to be attempted during payload generation. This also fixes tiny gpylint complaints. BUG=chromium:241283 TEST=Emitted partition files truncated as expected Change-Id: Ibb71e4f2305ec41224afdc503168ae02c312f6fe Reviewed-on: https://gerrit.chromium.org/gerrit/56532 Tested-by: Gilad Arnold Commit-Queue: Gilad Arnold Reviewed-by: Gilad Arnold --- scripts/update_payload/payload.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index 1796f511..0bac0915 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -155,7 +155,7 @@ class Payload(object): self.is_init = True def Describe(self): - + """Emits the payload embedded description data to standard output.""" def _DescribeImageInfo(description, image_info): def _DisplayIndentedValue(name, value): print ' {:<14} {}'.format(name+':', value) @@ -166,16 +166,18 @@ class Payload(object): _DisplayIndentedValue('Version', image_info.version) _DisplayIndentedValue('Key', image_info.key) - if (image_info.build_channel != image_info.channel): + if image_info.build_channel != image_info.channel: _DisplayIndentedValue('Build channel', image_info.build_channel) - if (image_info.build_version != image_info.version): + if image_info.build_version != image_info.version: _DisplayIndentedValue('Build version', image_info.build_version) if self.manifest.HasField('old_image_info'): + # pylint: disable=E1101 _DescribeImageInfo('Old Image', self.manifest.old_image_info) if self.manifest.HasField('new_image_info'): + # pylint: disable=E1101 _DescribeImageInfo('New Image', self.manifest.new_image_info) def _AssertInit(self): @@ -230,7 +232,8 @@ class Payload(object): report_out_file=report_out_file) def Apply(self, new_kernel_part, new_rootfs_part, old_kernel_part=None, - old_rootfs_part=None, bsdiff_in_place=True): + old_rootfs_part=None, bsdiff_in_place=True, + truncate_to_expected_size=True): """Applies the update payload. Args: @@ -239,6 +242,9 @@ class Payload(object): old_kernel_part: name of source kernel partition file (optional) old_rootfs_part: name of source rootfs partition file (optional) bsdiff_in_place: whether to perform BSDIFF operations in-place (optional) + truncate_to_expected_size: whether to truncate the resulting partitions + to their expected sizes, as specified in the + payload (optional) Raises: PayloadError if payload application failed. @@ -246,7 +252,9 @@ class Payload(object): self._AssertInit() # Create a short-lived payload applier object and run it. - helper = applier.PayloadApplier(self, bsdiff_in_place=bsdiff_in_place) + helper = applier.PayloadApplier( + self, bsdiff_in_place=bsdiff_in_place, + truncate_to_expected_size=truncate_to_expected_size) helper.Run(new_kernel_part, new_rootfs_part, old_kernel_part=old_kernel_part, old_rootfs_part=old_rootfs_part) -- cgit v1.2.3 From 21a0250e767dd6fc787252b9cc05657405332774 Mon Sep 17 00:00:00 2001 From: Gilad Arnold Date: Thu, 22 Aug 2013 16:59:48 -0700 Subject: paycheck: allow to pass an explicit path to bspatch The bspatch binary is used when applying update payloads. By default, we were using whatever bspatch that was found via path expansion in os.execvp, however there are cases where we want to be specific about where the bspatch binary is that we need to be using (such as during paygen runs). BUG=chromium:277072 TEST=Non-default bspatch binary used Change-Id: I85ffd28aeb26938cbf5ea428fa97d29af0353a7d Reviewed-on: https://gerrit.chromium.org/gerrit/66736 Tested-by: Gilad Arnold Reviewed-by: Don Garrett Commit-Queue: Gilad Arnold --- scripts/update_payload/payload.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index 0bac0915..b13aa111 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -232,7 +232,7 @@ class Payload(object): report_out_file=report_out_file) def Apply(self, new_kernel_part, new_rootfs_part, old_kernel_part=None, - old_rootfs_part=None, bsdiff_in_place=True, + old_rootfs_part=None, bsdiff_in_place=True, bspatch_path=None, truncate_to_expected_size=True): """Applies the update payload. @@ -242,6 +242,7 @@ class Payload(object): old_kernel_part: name of source kernel partition file (optional) old_rootfs_part: name of source rootfs partition file (optional) bsdiff_in_place: whether to perform BSDIFF operations in-place (optional) + bspatch_path: path to the bspatch binary (optional) truncate_to_expected_size: whether to truncate the resulting partitions to their expected sizes, as specified in the payload (optional) @@ -253,7 +254,7 @@ class Payload(object): # Create a short-lived payload applier object and run it. helper = applier.PayloadApplier( - self, bsdiff_in_place=bsdiff_in_place, + self, bsdiff_in_place=bsdiff_in_place, bspatch_path=bspatch_path, truncate_to_expected_size=truncate_to_expected_size) helper.Run(new_kernel_part, new_rootfs_part, old_kernel_part=old_kernel_part, -- cgit v1.2.3 From ef49735f40d0319151fd1e50abdbe377ab49ce72 Mon Sep 17 00:00:00 2001 From: Alex Deymo Date: Thu, 15 Oct 2015 09:14:58 -0700 Subject: Parse Payload v2 header. The update payload v2 contains an extra field in the header with the size of the metadata signatures and the metadata signatures stored right after the metadata. This patch parses the new payload format. BUG=b:22024447 TEST=cros payload show payload-v2.bin; served a payload v2 with devserver.py Change-Id: I8ce85af1df505f82f62a9d1cd57910cee6921f84 Reviewed-on: https://chromium-review.googlesource.com/306090 Commit-Ready: Alex Deymo Tested-by: Alex Deymo Reviewed-by: Gilad Arnold --- scripts/update_payload/payload.py | 98 ++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 22 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index b13aa111..ccd32400 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -49,14 +49,52 @@ class Payload(object): class _PayloadHeader(object): """Update payload header struct.""" - def __init__(self, version, manifest_len): - self.version = version - self.manifest_len = manifest_len + # Header constants; sizes are in bytes. + _MAGIC = 'CrAU' + _VERSION_SIZE = 8 + _MANIFEST_LEN_SIZE = 8 + _METADATA_SIGNATURE_LEN_SIZE = 4 + + def __init__(self): + self.version = None + self.manifest_len = None + self.metadata_signature_len = None + self.size = None + + def ReadFromPayload(self, payload_file, hasher=None): + """Reads the payload header from a file. + + Reads the payload header from the |payload_file| and updates the |hasher| + if one is passed. The parsed header is stored in the _PayloadHeader + instance attributes. + + Args: + payload_file: a file object + hasher: an optional hasher to pass the value through + Returns: + None. + Raises: + PayloadError if a read error occurred or the header is invalid. + """ + # Verify magic + magic = common.Read(payload_file, len(self._MAGIC), hasher=hasher) + if magic != self._MAGIC: + raise PayloadError('invalid payload magic: %s' % magic) + + self.version = _ReadInt(payload_file, self._VERSION_SIZE, True, + hasher=hasher) + self.manifest_len = _ReadInt(payload_file, self._MANIFEST_LEN_SIZE, True, + hasher=hasher) + self.size = (len(self._MAGIC) + self._VERSION_SIZE + + self._MANIFEST_LEN_SIZE) + self.metadata_signature_len = 0 + + if self.version == common.BRILLO_MAJOR_PAYLOAD_VERSION: + self.size += self._METADATA_SIGNATURE_LEN_SIZE + self.metadata_signature_len = _ReadInt( + payload_file, self._METADATA_SIGNATURE_LEN_SIZE, True, + hasher=hasher) - # Header constants; sizes are in bytes. - _MAGIC = 'CrAU' - _VERSION_SIZE = 8 - _MANIFEST_LEN_SIZE = 8 def __init__(self, payload_file): """Initialize the payload object. @@ -70,7 +108,9 @@ class Payload(object): self.is_init = False self.header = None self.manifest = None - self.data_offset = 0 + self.data_offset = None + self.metadata_signature = None + self.metadata_size = None def _ReadHeader(self): """Reads and returns the payload header. @@ -81,17 +121,9 @@ class Payload(object): PayloadError if a read error occurred. """ - # Verify magic - magic = common.Read(self.payload_file, len(self._MAGIC), - hasher=self.manifest_hasher) - if magic != self._MAGIC: - raise PayloadError('invalid payload magic: %s' % magic) - - return self._PayloadHeader( - _ReadInt(self.payload_file, self._VERSION_SIZE, True, - hasher=self.manifest_hasher), - _ReadInt(self.payload_file, self._MANIFEST_LEN_SIZE, True, - hasher=self.manifest_hasher)) + header = self._PayloadHeader() + header.ReadFromPayload(self.payload_file, self.manifest_hasher) + return header def _ReadManifest(self): """Reads and returns the payload manifest. @@ -108,6 +140,23 @@ class Payload(object): return common.Read(self.payload_file, self.header.manifest_len, hasher=self.manifest_hasher) + def _ReadMetadataSignature(self): + """Reads and returns the metadata signatures. + + Returns: + A string containing the metadata signatures protobuf in binary form or + an empty string if no metadata signature found in the payload. + Raises: + PayloadError if a read error occurred. + + """ + if not self.header: + raise PayloadError('payload header not present') + + return common.Read( + self.payload_file, self.header.metadata_signature_len, + offset=self.header.size + self.header.manifest_len) + def ReadDataBlob(self, offset, length): """Reads and returns a single data blob from the update payload. @@ -148,9 +197,14 @@ class Payload(object): self.manifest = update_metadata_pb2.DeltaArchiveManifest() self.manifest.ParseFromString(manifest_raw) - # Store data offset. - self.data_offset = (len(self._MAGIC) + self._VERSION_SIZE + - self._MANIFEST_LEN_SIZE + self.header.manifest_len) + # Read the metadata signature (if any). + metadata_signature_raw = self._ReadMetadataSignature() + if metadata_signature_raw: + self.metadata_signature = update_metadata_pb2.Signatures() + self.metadata_signature.ParseFromString(metadata_signature_raw) + + self.metadata_size = self.header.size + self.header.manifest_len + self.data_offset = self.metadata_size + self.header.metadata_signature_len self.is_init = True -- cgit v1.2.3 From 349fd298f4721e359ad598b6067a7254bb137adb Mon Sep 17 00:00:00 2001 From: Sen Jiang Date: Mon, 16 Nov 2015 17:28:09 -0800 Subject: update_payload: Fix IsDelta to support payload v2. Added check for the old_partition_info field which is used by major version 2 payload. Other changes are needed in order to pass presubmit lint check. BUG=None TEST=Run devserver with a v2 delta payload. Change-Id: I3d7057fde44bd24d768177d5ed5c36bc2d4ca94b Reviewed-on: https://chromium-review.googlesource.com/312952 Commit-Ready: Sen Jiang Tested-by: Sen Jiang Reviewed-by: Alex Deymo Reviewed-by: Gilad Arnold --- scripts/update_payload/payload.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) (limited to 'scripts/update_payload/payload.py') diff --git a/scripts/update_payload/payload.py b/scripts/update_payload/payload.py index ccd32400..f76c0de6 100644 --- a/scripts/update_payload/payload.py +++ b/scripts/update_payload/payload.py @@ -4,6 +4,8 @@ """Tools for reading, verifying and applying Chrome OS update payloads.""" +from __future__ import print_function + import hashlib import struct @@ -30,11 +32,12 @@ def _ReadInt(file_obj, size, is_unsigned, hasher=None): size: the integer size in bytes (2, 4 or 8) is_unsigned: whether it is signed or not hasher: an optional hasher to pass the value through + Returns: An "unpacked" (Python) integer value. + Raises: PayloadError if an read error occurred. - """ return struct.unpack(common.IntPackingFmtStr(size, is_unsigned), common.Read(file_obj, size, hasher=hasher))[0] @@ -71,8 +74,10 @@ class Payload(object): Args: payload_file: a file object hasher: an optional hasher to pass the value through + Returns: None. + Raises: PayloadError if a read error occurred or the header is invalid. """ @@ -101,7 +106,6 @@ class Payload(object): Args: payload_file: update payload file object open for reading - """ self.payload_file = payload_file self.manifest_hasher = None @@ -117,9 +121,9 @@ class Payload(object): Returns: A payload header object. + Raises: PayloadError if a read error occurred. - """ header = self._PayloadHeader() header.ReadFromPayload(self.payload_file, self.manifest_hasher) @@ -130,9 +134,9 @@ class Payload(object): Returns: A string containing the payload manifest in binary form. + Raises: PayloadError if a read error occurred. - """ if not self.header: raise PayloadError('payload header not present') @@ -146,9 +150,9 @@ class Payload(object): Returns: A string containing the metadata signatures protobuf in binary form or an empty string if no metadata signature found in the payload. + Raises: PayloadError if a read error occurred. - """ if not self.header: raise PayloadError('payload header not present') @@ -163,11 +167,12 @@ class Payload(object): Args: offset: offset to the beginning of the blob from the end of the manifest length: the blob's length + Returns: A string containing the raw blob data. + Raises: PayloadError if a read error occurred. - """ return common.Read(self.payload_file, length, offset=self.data_offset + offset) @@ -180,7 +185,6 @@ class Payload(object): Raises: PayloadError if object already initialized or fails to initialize correctly. - """ if self.is_init: raise PayloadError('payload object already initialized') @@ -212,9 +216,9 @@ class Payload(object): """Emits the payload embedded description data to standard output.""" def _DescribeImageInfo(description, image_info): def _DisplayIndentedValue(name, value): - print ' {:<14} {}'.format(name+':', value) + print(' {:<14} {}'.format(name+':', value)) - print '%s:' % description + print('%s:' % description) _DisplayIndentedValue('Channel', image_info.channel) _DisplayIndentedValue('Board', image_info.board) _DisplayIndentedValue('Version', image_info.version) @@ -247,7 +251,9 @@ class Payload(object): """Returns True iff the payload appears to be a delta.""" self._AssertInit() return (self.manifest.HasField('old_kernel_info') or - self.manifest.HasField('old_rootfs_info')) + self.manifest.HasField('old_rootfs_info') or + any(partition.HasField('old_partition_info') + for partition in self.manifest.partitions)) def IsFull(self): """Returns True iff the payload appears to be a full.""" @@ -269,9 +275,9 @@ class Payload(object): kernel_part_size: the size of (physical) kernel partitions in bytes allow_unhashed: allow unhashed operation blobs disabled_tests: list of tests to disable + Raises: PayloadError if payload verification failed. - """ self._AssertInit() @@ -300,9 +306,9 @@ class Payload(object): truncate_to_expected_size: whether to truncate the resulting partitions to their expected sizes, as specified in the payload (optional) + Raises: PayloadError if payload application failed. - """ self._AssertInit() @@ -327,7 +333,6 @@ class Payload(object): skip: the number of first origin mappings to skip trace_out_file: file object to dump the trace to is_kernel: trace through kernel (True) or rootfs (False) operations - """ self._AssertInit() -- cgit v1.2.3