Merge remote-tracking branch 'remotes/kevin/tags/for-upstream' into staging

Block layer patches

# gpg: Signature made Tue 06 Sep 2016 11:38:01 BST
# gpg:                using RSA key 0x7F09B272C88F2FD6
# gpg: Good signature from "Kevin Wolf <kwolf@redhat.com>"
# Primary key fingerprint: DC3D EB15 9A9A F95D 3D74  56FE 7F09 B272 C88F 2FD6

* remotes/kevin/tags/for-upstream: (36 commits)
  block: Allow node name for 'qemu-io' HMP command
  qemu-iotests: Log QMP traffic in debug mode
  block jobs: Improve error message for missing job ID
  coroutine: Assert that no locks are held on termination
  coroutine: Let CoMutex remember who holds it
  qcow2: fix iovec size at qcow2_co_pwritev_compressed
  test-coroutine: Fix coroutine pool corruption
  qemu-iotests: add vmdk for test backup compression in 055
  qemu-iotests: test backup compression in 055
  blockdev-backup: added support for data compression
  drive-backup: added support for data compression
  block: simplify blockdev-backup
  block: simplify drive-backup
  block/io: turn on dirty_bitmaps for the compressed writes
  block: remove BlockDriver.bdrv_write_compressed
  qcow: cleanup qcow_co_pwritev_compressed to avoid the recursion
  qcow: add qcow_co_pwritev_compressed
  vmdk: add vmdk_co_pwritev_compressed
  qcow2: cleanup qcow2_co_pwritev_compressed to avoid the recursion
  qcow2: add qcow2_co_pwritev_compressed
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

tests/qemu-iotests/030

@@ -126,7 +126,7 @@ class TestSingleDrive(iotests.QMPTestCase):
 
     def test_device_not_found(self):
         result = self.vm.qmp('block-stream', device='nonexistent')
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
 
 class TestSmallerBackingFile(iotests.QMPTestCase):

tests/qemu-iotests/041

@@ -38,7 +38,6 @@ class TestSingleDrive(iotests.QMPTestCase):
     image_len = 1 * 1024 * 1024 # MB
     qmp_cmd = 'drive-mirror'
     qmp_target = target_img
-    not_found_error = 'DeviceNotFound'
 
     def setUp(self):
         iotests.create_image(backing_img, self.image_len)
@@ -176,7 +175,7 @@ class TestSingleDrive(iotests.QMPTestCase):
 
         result = self.vm.qmp(self.qmp_cmd, device='ide1-cd0', sync='full',
                              target=self.qmp_target)
-        self.assert_qmp(result, 'error/class', self.not_found_error)
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_image_not_found(self):
         result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
@@ -186,12 +185,11 @@ class TestSingleDrive(iotests.QMPTestCase):
     def test_device_not_found(self):
         result = self.vm.qmp(self.qmp_cmd, device='nonexistent', sync='full',
                              target=self.qmp_target)
-        self.assert_qmp(result, 'error/class', self.not_found_error)
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
 class TestSingleBlockdev(TestSingleDrive):
     qmp_cmd = 'blockdev-mirror'
     qmp_target = 'node1'
-    not_found_error = 'GenericError'
 
     def setUp(self):
         TestSingleDrive.setUp(self)
@@ -922,7 +920,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
                              node_name='repair0',
                              replaces='img1',
                              target=quorum_repair_img, format=iotests.imgfmt)
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_wrong_sync_mode(self):
         if not self.has_quorum():

tests/qemu-iotests/055

@@ -134,10 +134,7 @@ class TestSingleDrive(iotests.QMPTestCase):
 
     def do_test_device_not_found(self, cmd, **args):
         result = self.vm.qmp(cmd, **args)
-        if cmd == 'drive-backup':
-            self.assert_qmp(result, 'error/class', 'DeviceNotFound')
-        else:
-            self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_device_not_found(self):
         self.do_test_device_not_found('drive-backup', device='nonexistent',
@@ -371,7 +368,7 @@ class TestSingleTransaction(iotests.QMPTestCase):
                           'sync': 'full' },
             }
         ])
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
         result = self.vm.qmp('transaction', actions=[{
                 'type': 'blockdev-backup',
@@ -451,5 +448,123 @@ class TestSingleTransaction(iotests.QMPTestCase):
         self.assert_qmp(result, 'error/class', 'GenericError')
         self.assert_no_active_block_jobs()
 
+
+class TestDriveCompression(iotests.QMPTestCase):
+    image_len = 64 * 1024 * 1024 # MB
+    fmt_supports_compression = [{'type': 'qcow2', 'args': ()},
+                                {'type': 'vmdk', 'args': ('-o', 'subformat=streamOptimized')}]
+
+    def setUp(self):
+        # Write data to the image so we can compare later
+        qemu_img('create', '-f', iotests.imgfmt, test_img, str(TestDriveCompression.image_len))
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x11 0 64k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x00 64k 128k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x22 162k 32k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x33 67043328 64k', test_img)
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(blockdev_target_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def do_prepare_drives(self, fmt, args):
+        self.vm = iotests.VM().add_drive(test_img)
+
+        qemu_img('create', '-f', fmt, blockdev_target_img,
+                 str(TestDriveCompression.image_len), *args)
+        self.vm.add_drive(blockdev_target_img, format=fmt)
+
+        self.vm.launch()
+
+    def do_test_compress_complete(self, cmd, format, **args):
+        self.do_prepare_drives(format['type'], format['args'])
+
+        self.assert_no_active_block_jobs()
+
+        result = self.vm.qmp(cmd, device='drive0', sync='full', compress=True, **args)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_until_completed()
+
+        self.vm.shutdown()
+        self.assertTrue(iotests.compare_images(test_img, blockdev_target_img,
+                                               iotests.imgfmt, format['type']),
+                        'target image does not match source after backup')
+
+    def test_complete_compress_drive_backup(self):
+        for format in TestDriveCompression.fmt_supports_compression:
+            self.do_test_compress_complete('drive-backup', format,
+                                           target=blockdev_target_img, mode='existing')
+
+    def test_complete_compress_blockdev_backup(self):
+        for format in TestDriveCompression.fmt_supports_compression:
+            self.do_test_compress_complete('blockdev-backup', format, target='drive1')
+
+    def do_test_compress_cancel(self, cmd, format, **args):
+        self.do_prepare_drives(format['type'], format['args'])
+
+        self.assert_no_active_block_jobs()
+
+        result = self.vm.qmp(cmd, device='drive0', sync='full', compress=True, **args)
+        self.assert_qmp(result, 'return', {})
+
+        event = self.cancel_and_wait()
+        self.assert_qmp(event, 'data/type', 'backup')
+
+        self.vm.shutdown()
+
+    def test_compress_cancel_drive_backup(self):
+        for format in TestDriveCompression.fmt_supports_compression:
+            self.do_test_compress_cancel('drive-backup', format,
+                                         target=blockdev_target_img, mode='existing')
+
+    def test_compress_cancel_blockdev_backup(self):
+       for format in TestDriveCompression.fmt_supports_compression:
+            self.do_test_compress_cancel('blockdev-backup', format, target='drive1')
+
+    def do_test_compress_pause(self, cmd, format, **args):
+        self.do_prepare_drives(format['type'], format['args'])
+
+        self.assert_no_active_block_jobs()
+
+        self.vm.pause_drive('drive0')
+        result = self.vm.qmp(cmd, device='drive0', sync='full', compress=True, **args)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-job-pause', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.resume_drive('drive0')
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        offset = self.dictpath(result, 'return[0]/offset')
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/offset', offset)
+
+        result = self.vm.qmp('block-job-resume', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_until_completed()
+
+        self.vm.shutdown()
+        self.assertTrue(iotests.compare_images(test_img, blockdev_target_img,
+                                               iotests.imgfmt, format['type']),
+                        'target image does not match source after backup')
+
+    def test_compress_pause_drive_backup(self):
+        for format in TestDriveCompression.fmt_supports_compression:
+            self.do_test_compress_pause('drive-backup', format,
+                                        target=blockdev_target_img, mode='existing')
+
+    def test_compress_pause_blockdev_backup(self):
+        for format in TestDriveCompression.fmt_supports_compression:
+            self.do_test_compress_pause('blockdev-backup', format, target='drive1')
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['raw', 'qcow2'])

tests/qemu-iotests/055.out

@@ -1,5 +1,5 @@
-........................
+..............................
 ----------------------------------------------------------------------
-Ran 24 tests
+Ran 30 tests
 
 OK

tests/qemu-iotests/057

@@ -182,7 +182,7 @@ class TestSingleTransaction(ImageSnapshotTestCase):
                               'name': 'a' },
                   }]
         result = self.vm.qmp('transaction', actions = actions)
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_error_exist(self):
         self.createSnapshotInTransaction(1)
@@ -239,7 +239,7 @@ class TestSnapshotDelete(ImageSnapshotTestCase):
         result = self.vm.qmp('blockdev-snapshot-delete-internal-sync',
                               device = 'drive_error',
                               id = '0')
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_error_no_id_and_name(self):
         result = self.vm.qmp('blockdev-snapshot-delete-internal-sync',

tests/qemu-iotests/iotests.py

@@ -50,6 +50,7 @@ cachemode = os.environ.get('CACHEMODE')
 qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE')
 
 socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
+debug = False
 
 def qemu_img(*args):
     '''Run qemu-img and return the exit code'''
@@ -86,10 +87,10 @@ def qemu_io(*args):
         sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args)))
     return subp.communicate()[0]
 
-def compare_images(img1, img2):
+def compare_images(img1, img2, fmt1=imgfmt, fmt2=imgfmt):
     '''Return True if two image files are identical'''
-    return qemu_img('compare', '-f', imgfmt,
-                    '-F', imgfmt, img1, img2) == 0
+    return qemu_img('compare', '-f', fmt1,
+                    '-F', fmt2, img1, img2) == 0
 
 def create_image(name, size):
     '''Create a fully-allocated raw image with sector markers'''
@@ -134,6 +135,8 @@ class VM(qtest.QEMUQtestMachine):
     def __init__(self):
         super(VM, self).__init__(qemu_prog, qemu_opts, test_dir=test_dir,
                                  socket_scm_helper=socket_scm_helper)
+        if debug:
+            self._debug = True
         self._num_drives = 0
 
     def add_drive_raw(self, opts):
@@ -141,14 +144,14 @@ class VM(qtest.QEMUQtestMachine):
         self._args.append(opts)
         return self
 
-    def add_drive(self, path, opts='', interface='virtio'):
+    def add_drive(self, path, opts='', interface='virtio', format=imgfmt):
         '''Add a virtio-blk drive to the VM'''
         options = ['if=%s' % interface,
                    'id=drive%d' % self._num_drives]
 
         if path is not None:
             options.append('file=%s' % path)
-            options.append('format=%s' % imgfmt)
+            options.append('format=%s' % format)
             options.append('cache=%s' % cachemode)
 
         if opts:
@@ -318,6 +321,8 @@ def verify_quorum():
 def main(supported_fmts=[], supported_oses=['linux']):
     '''Run tests'''
 
+    global debug
+
     # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to
     # indicate that we're not being run via "check". There may be
     # other things set up by "check" that individual test cases rely

tests/test-coroutine.c

@@ -139,13 +139,20 @@ static void test_co_queue(void)
 {
     Coroutine *c1;
     Coroutine *c2;
+    Coroutine tmp;
 
     c2 = qemu_coroutine_create(c2_fn, NULL);
     c1 = qemu_coroutine_create(c1_fn, c2);
 
     qemu_coroutine_enter(c1);
+
+    /* c1 shouldn't be used any more now; make sure we segfault if it is */
+    tmp = *c1;
     memset(c1, 0xff, sizeof(Coroutine));
     qemu_coroutine_enter(c2);
+
+    /* Must restore the coroutine now to avoid corrupted pool */
+    *c1 = tmp;
 }
 
 /*