Update version for 4.1.1 release

Signed-off-by: Michael Roth <mdroth@linux.vnet.ibm.com>
mirror: Keep mirror_top_bs drained after dropping permissions
2019-11-14 12:04:03 -06:00 · 2019-11-12 16:31:49 -06:00 · 2019-11-12 16:31:38 -06:00 · 2019-11-12 16:23:54 -06:00 · 2019-11-12 16:23:54 -06:00 · 2019-11-12 16:23:49 -06:00
2880 changed files with 37157 additions and 96740 deletions
@@ -25,14 +25,3 @@ macos_task:
    - ./configure --python=/usr/local/bin/python3 || { cat config.log; exit 1; }
    - gmake -j$(sysctl -n hw.ncpu)
    - gmake check -j$(sysctl -n hw.ncpu)
-
-macos_xcode_task:
-  osx_instance:
-    # this is an alias for the latest Xcode
-    image: mojave-xcode
-  install_script:
-    - brew install pkg-config gnu-sed glib pixman make sdl2
-  script:
-    - ./configure --cc=clang || { cat config.log; exit 1; }
-    - gmake -j$(sysctl -n hw.ncpu)
-    - gmake check -j$(sysctl -n hw.ncpu)
@@ -41,7 +41,6 @@
 /qapi/qapi-types-*.[ch]
 /qapi/qapi-types.[ch]
 /qapi/qapi-visit-*.[ch]
-!/qapi/qapi-visit-core.c
 /qapi/qapi-visit.[ch]
 /qapi/qapi-doc.texi
 /qemu-doc.html
@@ -66,8 +65,6 @@
 /scsi/qemu-pr-helper
 /vhost-user-scsi
 /vhost-user-blk
-/vhost-user-gpu
-/vhost-user-input
 /fsdev/virtfs-proxy-helper
 *.tmp
 *.[1-9]
@@ -134,7 +131,6 @@
 /docs/interop/qemu-qmp-ref.info*
 /docs/interop/qemu-qmp-ref.txt
 /docs/version.texi
-/contrib/vhost-user-gpu/50-qemu-gpu.json
 *.tps
 .stgit-*
 .git-submodule-status
@@ -5,7 +5,7 @@ before_script:
 build-system1:
 script:
 - apt-get install -y -qq libgtk-3-dev libvte-dev nettle-dev libcacard-dev
-      libusb-dev libvde-dev libspice-protocol-dev libgl1-mesa-dev libvdeplug-dev
+      libusb-dev libvde-dev libspice-protocol-dev libgl1-mesa-dev
 - ./configure --enable-werror --target-list="aarch64-softmmu alpha-softmmu
      cris-softmmu hppa-softmmu lm32-softmmu moxie-softmmu microblazeel-softmmu
      mips64el-softmmu m68k-softmmu ppc-softmmu riscv64-softmmu sparc-softmmu"
@@ -45,10 +45,15 @@ build-tcg-disabled:
 - ./check -raw 001 002 003 004 005 008 009 010 011 012 021 025 032 033 048
            052 063 077 086 101 104 106 113 147 148 150 151 152 157 159 160
            163 170 171 183 184 192 194 197 205 208 215 221 222 226 227 236
- - ./check -qcow2 028 040 051 056 057 058 065 067 068 082 085 091 095 096 102
-            122 124 127 129 132 139 142 144 145 147 151 152 155 157 165 194
-            196 197 200 202 203 205 208 209 215 216 218 222 227 234 246 247
-            248 250 254 255 256
+ - ./check -qcow2 001 002 003 004 005 007 008 009 010 011 012 013 017 018 019
+            020 021 022 024 025 027 028 029 031 032 033 034 035 036 037 038
+            039 040 042 043 046 047 048 049 050 051 052 053 054 056 057 058
+            060 061 062 063 065 066 067 068 069 071 072 073 074 079 080 082
+            085 086 089 090 091 095 096 097 098 099 102 103 104 105 107 108
+            110 111 114 117 120 122 124 126 127 129 130 132 133 134 137 138
+            139 140 141 142 143 144 145 147 150 151 152 154 155 156 157 158
+            161 165 170 172 174 176 177 179 184 186 187 190 192 194 195 196
+            197 200 202 203 205 208 209 214 215 216 217 218 222 226 227 229 234

 build-user:
 script:
@@ -59,7 +64,7 @@ build-user:

 build-clang:
 script:
- - apt-get install -y -qq clang libsdl2-dev libattr1-dev libcap-dev
+ - apt-get install -y -qq clang libsdl2-dev
      xfslibs-dev libiscsi-dev libnfs-dev libseccomp-dev gnutls-dev librbd-dev
 - ./configure --cc=clang --cxx=clang++ --enable-werror
      --target-list="alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu
@@ -58,6 +58,3 @@
 [submodule "roms/opensbi"]
 	path = roms/opensbi
 	url = 	https://git.qemu.org/git/opensbi.git
-[submodule "roms/qboot"]
-	path = roms/qboot
-	url = https://github.com/bonzini/qboot
@@ -1,29 +1,23 @@
 # This mailmap fixes up author names/addresses.
-#
-# If you are adding to this file consider if a similar change needs to
-# be made to contrib/gitdm/aliases. They are not however completely
-# analogous. .mailmap is concerned with fixing up damaged author
-# fields where as the gitdm equivalent is more concerned with making
-# sure multiple email addresses get mapped onto the same author.
-#
-# From man git-shortlog the forms are:
-#
-#  Proper Name <commit@email.xx>
-#  <proper@email.xx> <commit@email.xx>
-#  Proper Name <proper@email.xx> <commit@email.xx>
-#  Proper Name <proper@email.xx> Commit Name <commit@email.xx>
-#

 # The first section translates weird addresses from the original git import
 # into proper addresses so that they are counted properly by git shortlog.
 Andrzej Zaborowski <balrogg@gmail.com> balrog <balrog@c046a42c-6fe2-441c-8c8c-71466251a162>
 Anthony Liguori <anthony@codemonkey.ws> aliguori <aliguori@c046a42c-6fe2-441c-8c8c-71466251a162>
+Anthony Liguori <anthony@codemonkey.ws> Anthony Liguori <aliguori@us.ibm.com>
 Aurelien Jarno <aurelien@aurel32.net> aurel32 <aurel32@c046a42c-6fe2-441c-8c8c-71466251a162>
 Blue Swirl <blauwirbel@gmail.com> blueswir1 <blueswir1@c046a42c-6fe2-441c-8c8c-71466251a162>
 Edgar E. Iglesias <edgar.iglesias@gmail.com> edgar_igl <edgar_igl@c046a42c-6fe2-441c-8c8c-71466251a162>
 Fabrice Bellard <fabrice@bellard.org> bellard <bellard@c046a42c-6fe2-441c-8c8c-71466251a162>
+James Hogan <jhogan@kernel.org> <james.hogan@imgtec.com>
 Jocelyn Mayer <l_indien@magic.fr> j_mayer <j_mayer@c046a42c-6fe2-441c-8c8c-71466251a162>
 Paul Brook <paul@codesourcery.com> pbrook <pbrook@c046a42c-6fe2-441c-8c8c-71466251a162>
+Yongbok Kim <yongbok.kim@mips.com> <yongbok.kim@imgtec.com>
+Aleksandar Markovic <amarkovic@wavecomp.com> <aleksandar.markovic@mips.com>
+Aleksandar Markovic <amarkovic@wavecomp.com> <aleksandar.markovic@imgtec.com>
+Paul Burton <pburton@wavecomp.com> <paul.burton@mips.com>
+Paul Burton <pburton@wavecomp.com> <paul.burton@imgtec.com>
+Paul Burton <pburton@wavecomp.com> <paul@archlinuxmips.org>
 Thiemo Seufer <ths@networkno.de> ths <ths@c046a42c-6fe2-441c-8c8c-71466251a162>
 malc <av1474@comtv.ru> malc <malc@c046a42c-6fe2-441c-8c8c-71466251a162>

@@ -38,124 +32,8 @@ Ian McKellar <ianloic@google.com> Ian McKellar via Qemu-devel <qemu-devel@nongnu
 Julia Suvorova <jusual@mail.ru> Julia Suvorova via Qemu-devel <qemu-devel@nongnu.org>
 Justin Terry (VM) <juterry@microsoft.com> Justin Terry (VM) via Qemu-devel <qemu-devel@nongnu.org>

-# Next, replace old addresses by a more recent one.
-Aleksandar Markovic <amarkovic@wavecomp.com> <aleksandar.markovic@mips.com>
-Aleksandar Markovic <amarkovic@wavecomp.com> <aleksandar.markovic@imgtec.com>
-Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com> <arikalo@wavecomp.com>
-Anthony Liguori <anthony@codemonkey.ws> Anthony Liguori <aliguori@us.ibm.com>
-James Hogan <jhogan@kernel.org> <james.hogan@imgtec.com>
-Paul Burton <pburton@wavecomp.com> <paul.burton@mips.com>
-Paul Burton <pburton@wavecomp.com> <paul.burton@imgtec.com>
-Paul Burton <pburton@wavecomp.com> <paul@archlinuxmips.org>
-Philippe Mathieu-Daudé <philmd@redhat.com> <f4bug@amsat.org>
-Yongbok Kim <yongbok.kim@mips.com> <yongbok.kim@imgtec.com>

 # Also list preferred name forms where people have changed their
 # git author config, or had utf8/latin1 encoding issues.
-Aaron Lindsay <aaron@os.amperecomputing.com>
-Alexey Gerasimenko <x1917x@gmail.com>
-Alex Ivanov <void@aleksoft.net>
-Andreas Färber <afaerber@suse.de>
-Bandan Das <bsd@redhat.com>
-Benjamin MARSILI <mlspirat42@gmail.com>
-Benoît Canet <benoit.canet@gmail.com>
-Benoît Canet <benoit.canet@irqsave.net>
-Benoît Canet <benoit.canet@nodalink.com>
-Boqun Feng <boqun.feng@gmail.com>
-Boqun Feng <boqun.feng@intel.com>
-Brad Smith <brad@comstyle.com>
-Brijesh Singh <brijesh.singh@amd.com>
-Brilly Wu <brillywu@viatech.com.cn>
-Cédric Vincent <cedric.vincent@st.com>
-CheneyLin <linzc@zju.edu.cn>
-Chen Gang <chengang@emindsoft.com.cn>
-Chen Gang <gang.chen.5i5j@gmail.com>
-Chen Gang <gang.chen@sunrus.com.cn>
-Chen Wei-Ren <chenwj@iis.sinica.edu.tw>
-Christophe Lyon <christophe.lyon@st.com>
-Collin L. Walling <walling@linux.ibm.com>
 Daniel P. Berrangé <berrange@redhat.com>
-Eduardo Otubo <otubo@redhat.com>
-Fabrice Desclaux <fabrice.desclaux@cea.fr>
-Fernando Luis Vázquez Cao <fernando_b1@lab.ntt.co.jp>
-Fernando Luis Vázquez Cao <fernando@oss.ntt.co.jp>
-Gautham R. Shenoy <ego@in.ibm.com>
-Gautham R. Shenoy <ego@linux.vnet.ibm.com>
-Gonglei (Arei) <arei.gonglei@huawei.com>
-Guang Wang <wang.guang55@zte.com.cn>
-Hailiang Zhang <zhang.zhanghailiang@huawei.com>
-Hervé Poussineau <hpoussin@reactos.org>
-Jakub Jermář <jakub@jermar.eu>
-Jakub Jermář <jakub.jermar@kernkonzept.com>
-Jean-Christophe Dubois <jcd@tribudubois.net>
-Jindřich Makovička <makovick@gmail.com>
-John Arbuckle <programmingkidx@gmail.com>
-Juha Riihimäki <juha.riihimaki@nokia.com>
-Juha Riihimäki <Juha.Riihimaki@nokia.com>
-Jun Li <junmuzi@gmail.com>
-Laurent Vivier <Laurent@lvivier.info>
-Leandro Lupori <leandro.lupori@gmail.com>
-Li Guang <lig.fnst@cn.fujitsu.com>
-Liming Wang <walimisdev@gmail.com>
-linzhecheng <linzc@zju.edu.cn>
-Liran Schour <lirans@il.ibm.com>
-Liu Yu <yu.liu@freescale.com>
-Liu Yu <Yu.Liu@freescale.com>
-Li Zhang <zhlcindy@gmail.com>
-Li Zhang <zhlcindy@linux.vnet.ibm.com>
-Lluís Vilanova <vilanova@ac.upc.edu>
-Lluís Vilanova <xscript@gmx.net>
-Longpeng (Mike) <longpeng2@huawei.com>
-Luc Michel <luc.michel@git.antfield.fr>
-Luc Michel <luc.michel@greensocs.com>
-Marc Marí <marc.mari.barcelo@gmail.com>
-Marc Marí <markmb@redhat.com>
-Michael Avdienko <whitearchey@gmail.com>
-Michael S. Tsirkin <mst@redhat.com>
-Munkyu Im <munkyu.im@samsung.com>
-Nicholas Bellinger <nab@linux-iscsi.org>
-Nicholas Thomas <nick@bytemark.co.uk>
-Nikunj A Dadhania <nikunj@linux.vnet.ibm.com>
-Orit Wasserman <owasserm@redhat.com>
-Paolo Bonzini <pbonzini@redhat.com>
-Pavel Dovgaluk <dovgaluk@ispras.ru>
-Pavel Dovgaluk <pavel.dovgaluk@gmail.com>
-Pavel Dovgaluk <Pavel.Dovgaluk@ispras.ru>
-Peter Crosthwaite <crosthwaite.peter@gmail.com>
-Peter Crosthwaite <peter.crosthwaite@petalogix.com>
-Peter Crosthwaite <peter.crosthwaite@xilinx.com>
-Prasad J Pandit <pjp@fedoraproject.org>
-Prasad J Pandit <ppandit@redhat.com>
-Qiao Nuohan <qiaonuohan@cn.fujitsu.com>
 Reimar Döffinger <Reimar.Doeffinger@gmx.de>
-Remy Noel <remy.noel@blade-group.com>
-Roger Pau Monné <roger.pau@citrix.com>
-Shin'ichiro Kawasaki <kawasaki@juno.dti.ne.jp>
-Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>
-Sochin Jiang <sochin.jiang@huawei.com>
-Takashi Yoshii <takasi-y@ops.dti.ne.jp>
-Thomas Huth <thuth@redhat.com>
-Thomas Knych <thomaswk@google.com>
-Timothy Baldwin <T.E.Baldwin99@members.leeds.ac.uk>
-Tony Nguyen <tony.nguyen@bt.com>
-Venkateswararao Jujjuri <jvrao@linux.vnet.ibm.com>
-Vibi Sreenivasan <vibi_sreenivasan@cms.com>
-Vijaya Kumar K <vijayak@cavium.com>
-Vijaya Kumar K <Vijaya.Kumar@cavium.com>
-Vijay Kumar <vijaykumar@bravegnu.org>
-Vijay Kumar <vijaykumar@zilogic.com>
-Wang Guang <wang.guang55@zte.com.cn>
-Wenchao Xia <xiawenc@linux.vnet.ibm.com>
-Wenshuang Ma <kevinnma@tencent.com>
-Xiaoqiang Zhao <zxq_yx_007@163.com>
-Xinhua Cao <caoxinhua@huawei.com>
-Xiong Zhang <xiong.y.zhang@intel.com>
-Yin Yin <yin.yin@cs2c.com.cn>
-yuchenlin <npes87184@gmail.com>
-YunQiang Su <syq@debian.org>
-YunQiang Su <ysu@wavecomp.com>
-Yuri Pudgorodskiy <yur@virtuozzo.com>
-Zhengui Li <lizhengui@huawei.com>
-Zhenwei Pi <pizhenwei@bytedance.com>
-Zhenwei Pi <zhenwei.pi@youruncloud.com>
-Zhuang Yanying <ann.zhuangyanying@huawei.com>
@@ -5,18 +5,7 @@ dist: xenial
 language: c
 compiler:
  - gcc
-cache:
-  # There is one cache per branch and compiler version.
-  # characteristics of each job are used to identify the cache:
-  # - OS name (currently, linux, osx, or windows)
-  # - OS distribution (for Linux, xenial, trusty, or precise)
-  # - macOS image name (e.g., xcode7.2)
-  # - Names and values of visible environment variables set in .travis.yml or Settings panel
-  timeout: 1200
-  ccache: true
-  pip: true
-  directories:
-  - $HOME/avocado/data/cache
+cache: ccache


 addons:
@@ -26,10 +15,9 @@ addons:
      - libaio-dev
      - libattr1-dev
      - libbrlapi-dev
-      - libcap-dev
      - libcap-ng-dev
      - libgcc-4.8-dev
-      - libgnutls28-dev
+      - libgnutls-dev
      - libgtk-3-dev
      - libiscsi-dev
      - liblttng-ust-dev
@@ -37,24 +25,21 @@ addons:
      - libnfs-dev
      - libnss3-dev
      - libpixman-1-dev
-      - libpng-dev
+      - libpng12-dev
      - librados-dev
-      - libsdl2-dev
-      - libsdl2-image-dev
+      - libsdl1.2-dev
      - libseccomp-dev
      - libspice-protocol-dev
      - libspice-server-dev
      - libssh-dev
      - liburcu-dev
      - libusb-1.0-0-dev
-      - libvdeplug-dev
      - libvte-2.91-dev
      - sparse
      - uuid-dev
      - gcovr
  homebrew:
    packages:
-      - ccache
      - glib
      - pixman
      - gnu-sed
@@ -80,9 +65,6 @@ env:
    - TEST_CMD="make check -j3 V=1"
    # This is broadly a list of "mainline" softmmu targets which have support across the major distros
    - MAIN_SOFTMMU_TARGETS="aarch64-softmmu,arm-softmmu,i386-softmmu,mips-softmmu,mips64-softmmu,ppc64-softmmu,riscv64-softmmu,s390x-softmmu,x86_64-softmmu"
-    - CCACHE_SLOPPINESS="include_file_ctime,include_file_mtime"
-    - CCACHE_MAXSIZE=1G
-

 git:
  # we want to do this ourselves
@@ -90,32 +72,25 @@ git:


 before_script:
-  - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then export PATH="/usr/local/opt/ccache/libexec:$PATH" ; fi
-  - if command -v ccache ; then ccache --zero-stats ; fi
  - mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR}
  - ${SRC_DIR}/configure ${BASE_CONFIG} ${CONFIG} || { cat config.log && exit 1; }
 script:
  - make -j3 && travis_retry ${TEST_CMD}
-after_script:
-  - if command -v ccache ; then ccache --show-stats ; fi


 matrix:
  include:
    - env:
        - CONFIG="--disable-system --static"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"


    # we split the system builds as it takes a while to build them all
    - env:
        - CONFIG="--disable-user --target-list=${MAIN_SOFTMMU_TARGETS}"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"


    - env:
        - CONFIG="--disable-user --target-list-exclude=${MAIN_SOFTMMU_TARGETS}"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"


    # Just build tools and run minimal unit and softfloat checks
@@ -123,19 +98,14 @@ matrix:
        - BASE_CONFIG="--enable-tools"
        - CONFIG="--disable-user --disable-system"
        - TEST_CMD="make check-unit check-softfloat -j3"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"

-
-    # --enable-debug implies --enable-debug-tcg, also runs quite a bit slower
    - env:
-        - CONFIG="--enable-debug --target-list=${MAIN_SOFTMMU_TARGETS}"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-debug"
+        - CONFIG="--enable-debug --enable-debug-tcg --disable-user"


-    # TCG debug can be run just on its own and is mostly agnostic to user/softmmu distinctions
+    # TCG debug can be run just on it's own and is mostly agnostic to user/softmmu distinctions
    - env:
        - CONFIG="--enable-debug-tcg --disable-system"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-debug-tcg"


    - env:
@@ -145,7 +115,6 @@ matrix:
    # Module builds are mostly of interest to major distros
    - env:
        - CONFIG="--enable-modules --target-list=${MAIN_SOFTMMU_TARGETS}"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"


    # Alternate coroutines implementations are only really of interest to KVM users
@@ -165,7 +134,6 @@ matrix:
        - BUILD_DIR="out-of-tree/build/dir" SRC_DIR="../../.."
        - BASE_CONFIG="--enable-tools --enable-docs"
        - CONFIG="--target-list=x86_64-softmmu,aarch64-linux-user"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"
      addons:
        apt:
          packages:
@@ -177,19 +145,16 @@ matrix:
    # Test with Clang for compile portability (Travis uses clang-5.0)
    - env:
        - CONFIG="--disable-system"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-clang-default"
      compiler: clang


    - env:
        - CONFIG="--disable-user --target-list=${MAIN_SOFTMMU_TARGETS}"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-clang-default"
      compiler: clang


    - env:
        - CONFIG="--target-list=${MAIN_SOFTMMU_TARGETS} "
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-clang-sanitize"
      compiler: clang
      before_script:
        - ./configure ${CONFIG} --extra-cflags="-fsanitize=undefined -Werror" || { cat config.log && exit 1; }
@@ -197,7 +162,6 @@ matrix:

    - env:
        - CONFIG="--disable-user --target-list-exclude=${MAIN_SOFTMMU_TARGETS}"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-clang-default"
      compiler: clang


@@ -211,7 +175,6 @@ matrix:
    # We manually include builds which we disable "make check" for
    - env:
        - CONFIG="--without-default-devices --disable-user"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"
        - TEST_CMD=""


@@ -248,14 +211,13 @@ matrix:
    - env:
        - CONFIG="--target-list=i386-softmmu,ppc-softmmu,ppc64-softmmu,m68k-softmmu,x86_64-softmmu"
      os: osx
-      osx_image: xcode10.3
+      osx_image: xcode10.2
      compiler: clang


    # Python builds
    - env:
        - CONFIG="--target-list=x86_64-softmmu"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"
      language: python
      python:
        - "3.4"
@@ -263,7 +225,6 @@ matrix:

    - env:
        - CONFIG="--target-list=x86_64-softmmu"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"
      language: python
      python:
        - "3.6"
@@ -271,20 +232,15 @@ matrix:

    # Acceptance (Functional) tests
    - env:
-        - CONFIG="--python=/usr/bin/python3 --target-list=x86_64-softmmu,mips-softmmu,mips64el-softmmu,aarch64-softmmu,arm-softmmu,s390x-softmmu,alpha-softmmu,ppc-softmmu,ppc64-softmmu,m68k-softmmu,sparc-softmmu"
+        - CONFIG="--python=/usr/bin/python3 --target-list=x86_64-softmmu,mips-softmmu,mips64el-softmmu,aarch64-softmmu,arm-softmmu,s390x-softmmu,alpha-softmmu"
        - TEST_CMD="make check-acceptance"
      after_failure:
        - cat tests/results/latest/job.log
      addons:
        apt:
          packages:
-            - python3-pil
            - python3-pip
            - python3.5-venv
-            - tesseract-ocr
-            - tesseract-ocr-eng
-
-
    # Using newer GCC with sanitizers
    - addons:
        apt:
@@ -311,8 +267,7 @@ matrix:
            - libpixman-1-dev
            - libpng12-dev
            - librados-dev
-            - libsdl2-dev
-            - libsdl2-image-dev
+            - libsdl1.2-dev
            - libseccomp-dev
            - libspice-protocol-dev
            - libspice-server-dev
@@ -334,51 +289,10 @@ matrix:

    # Run check-tcg against linux-user
    - env:
-        - CONFIG="--disable-system --enable-debug-tcg"
+        - CONFIG="--disable-system"
        - TEST_CMD="make -j3 check-tcg V=1"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-debug-tcg"
-
-
-    # Run check-tcg against linux-user (with plugins)
-    # we skip sparc64-linux-user until it has been fixed somewhat
-    - env:
-        - CONFIG="--disable-system --enable-plugins --enable-debug-tcg --target-list-exclude=sparc64-linux-user"
-        - TEST_CMD="make -j3 check-tcg V=1"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-debug-tcg"
-

    # Run check-tcg against softmmu targets
    - env:
-        - CONFIG="--enable-debug-tcg --target-list=xtensa-softmmu,arm-softmmu,aarch64-softmmu,alpha-softmmu"
+        - CONFIG="--target-list=xtensa-softmmu,arm-softmmu,aarch64-softmmu,alpha-softmmu"
        - TEST_CMD="make -j3 check-tcg V=1"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-debug-tcg"
-
-
-    # Run check-tcg against softmmu targets (with plugins)
-    - env:
-        - CONFIG="--enable-plugins --enable-debug-tcg --target-list=xtensa-softmmu,arm-softmmu,aarch64-softmmu,alpha-softmmu"
-        - TEST_CMD="make -j3 check-tcg V=1"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-debug-tcg"
-
-
-    # Release builds
-    # The make-release script expect a QEMU version, so our tag must start with a 'v'.
-    # This is the case when release candidate tags are created.
-    - if: tag IS present AND tag =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/
-      env:
-        # We want to build from the release tarball
-        - BUILD_DIR="release/build/dir" SRC_DIR="../../.."
-        - BASE_CONFIG="--prefix=$PWD/dist"
-        - CONFIG="--target-list=x86_64-softmmu,aarch64-softmmu,armeb-linux-user,ppc-linux-user"
-        - TEST_CMD="make install -j3"
-        - QEMU_VERSION="${TRAVIS_TAG:1}"
-        - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-default"
-      before_script:
-        - command -v ccache && ccache --zero-stats
-        - mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR}
-      script:
-        - make -C ${SRC_DIR} qemu-${QEMU_VERSION}.tar.bz2
-        - ls -l ${SRC_DIR}/qemu-${QEMU_VERSION}.tar.bz2
-        - tar -xf ${SRC_DIR}/qemu-${QEMU_VERSION}.tar.bz2 && cd qemu-${QEMU_VERSION}
-        - ./configure ${BASE_CONFIG} ${CONFIG} || { cat config.log && exit 1; }
-        - make install
@@ -0,0 +1,216 @@
+QEMU Coding Style
+=================
+
+Please use the script checkpatch.pl in the scripts directory to check
+patches before submitting.
+
+1. Whitespace
+
+Of course, the most important aspect in any coding style is whitespace.
+Crusty old coders who have trouble spotting the glasses on their noses
+can tell the difference between a tab and eight spaces from a distance
+of approximately fifteen parsecs.  Many a flamewar has been fought and
+lost on this issue.
+
+QEMU indents are four spaces.  Tabs are never used, except in Makefiles
+where they have been irreversibly coded into the syntax.
+Spaces of course are superior to tabs because:
+
+ - You have just one way to specify whitespace, not two.  Ambiguity breeds
+   mistakes.
+ - The confusion surrounding 'use tabs to indent, spaces to justify' is gone.
+ - Tab indents push your code to the right, making your screen seriously
+   unbalanced.
+ - Tabs will be rendered incorrectly on editors who are misconfigured not
+   to use tab stops of eight positions.
+ - Tabs are rendered badly in patches, causing off-by-one errors in almost
+   every line.
+ - It is the QEMU coding style.
+
+Do not leave whitespace dangling off the ends of lines.
+
+1.1 Multiline Indent
+
+There are several places where indent is necessary:
+
+ - if/else
+ - while/for
+ - function definition & call
+
+When breaking up a long line to fit within line width, we need a proper indent
+for the following lines.
+
+In case of if/else, while/for, align the secondary lines just after the
+opening parenthesis of the first.
+
+For example:
+
+    if (a == 1 &&
+        b == 2) {
+
+    while (a == 1 &&
+           b == 2) {
+
+In case of function, there are several variants:
+
+    * 4 spaces indent from the beginning
+    * align the secondary lines just after the opening parenthesis of the
+      first
+
+For example:
+
+    do_something(x, y,
+        z);
+
+    do_something(x, y,
+                 z);
+
+    do_something(x, do_another(y,
+                               z));
+
+2. Line width
+
+Lines should be 80 characters; try not to make them longer.
+
+Sometimes it is hard to do, especially when dealing with QEMU subsystems
+that use long function or symbol names.  Even in that case, do not make
+lines much longer than 80 characters.
+
+Rationale:
+ - Some people like to tile their 24" screens with a 6x4 matrix of 80x24
+   xterms and use vi in all of them.  The best way to punish them is to
+   let them keep doing it.
+ - Code and especially patches is much more readable if limited to a sane
+   line length.  Eighty is traditional.
+ - The four-space indentation makes the most common excuse ("But look
+   at all that white space on the left!") moot.
+ - It is the QEMU coding style.
+
+3. Naming
+
+Variables are lower_case_with_underscores; easy to type and read.  Structured
+type names are in CamelCase; harder to type but standing out.  Enum type
+names and function type names should also be in CamelCase.  Scalar type
+names are lower_case_with_underscores_ending_with_a_t, like the POSIX
+uint64_t and family.  Note that this last convention contradicts POSIX
+and is therefore likely to be changed.
+
+When wrapping standard library functions, use the prefix qemu_ to alert
+readers that they are seeing a wrapped version; otherwise avoid this prefix.
+
+4. Block structure
+
+Every indented statement is braced; even if the block contains just one
+statement.  The opening brace is on the line that contains the control
+flow statement that introduces the new block; the closing brace is on the
+same line as the else keyword, or on a line by itself if there is no else
+keyword.  Example:
+
+    if (a == 5) {
+        printf("a was 5.\n");
+    } else if (a == 6) {
+        printf("a was 6.\n");
+    } else {
+        printf("a was something else entirely.\n");
+    }
+
+Note that 'else if' is considered a single statement; otherwise a long if/
+else if/else if/.../else sequence would need an indent for every else
+statement.
+
+An exception is the opening brace for a function; for reasons of tradition
+and clarity it comes on a line by itself:
+
+    void a_function(void)
+    {
+        do_something();
+    }
+
+Rationale: a consistent (except for functions...) bracing style reduces
+ambiguity and avoids needless churn when lines are added or removed.
+Furthermore, it is the QEMU coding style.
+
+5. Declarations
+
+Mixed declarations (interleaving statements and declarations within
+blocks) are generally not allowed; declarations should be at the beginning
+of blocks.
+
+Every now and then, an exception is made for declarations inside a
+#ifdef or #ifndef block: if the code looks nicer, such declarations can
+be placed at the top of the block even if there are statements above.
+On the other hand, however, it's often best to move that #ifdef/#ifndef
+block to a separate function altogether.
+
+6. Conditional statements
+
+When comparing a variable for (in)equality with a constant, list the
+constant on the right, as in:
+
+    if (a == 1) {
+        /* Reads like: "If a equals 1" */
+        do_something();
+    }
+
+Rationale: Yoda conditions (as in 'if (1 == a)') are awkward to read.
+Besides, good compilers already warn users when '==' is mis-typed as '=',
+even when the constant is on the right.
+
+7. Comment style
+
+We use traditional C-style /* */ comments and avoid // comments.
+
+Rationale: The // form is valid in C99, so this is purely a matter of
+consistency of style. The checkpatch script will warn you about this.
+
+Multiline comment blocks should have a row of stars on the left,
+and the initial /* and terminating */ both on their own lines:
+    /*
+     * like
+     * this
+     */
+This is the same format required by the Linux kernel coding style.
+
+(Some of the existing comments in the codebase use the GNU Coding
+Standards form which does not have stars on the left, or other
+variations; avoid these when writing new comments, but don't worry
+about converting to the preferred form unless you're editing that
+comment anyway.)
+
+Rationale: Consistency, and ease of visually picking out a multiline
+comment from the surrounding code.
+
+8. trace-events style
+
+8.1 0x prefix
+
+In trace-events files, use a '0x' prefix to specify hex numbers, as in:
+
+some_trace(unsigned x, uint64_t y) "x 0x%x y 0x" PRIx64
+
+An exception is made for groups of numbers that are hexadecimal by
+convention and separated by the symbols '.', '/', ':', or ' ' (such as
+PCI bus id):
+
+another_trace(int cssid, int ssid, int dev_num) "bus id: %x.%x.%04x"
+
+However, you can use '0x' for such groups if you want. Anyway, be sure that
+it is obvious that numbers are in hex, ex.:
+
+data_dump(uint8_t c1, uint8_t c2, uint8_t c3) "bytes (in hex): %02x %02x %02x"
+
+Rationale: hex numbers are hard to read in logs when there is no 0x prefix,
+especially when (occasionally) the representation doesn't contain any letters
+and especially in one line with other decimal numbers. Number groups are allowed
+to not use '0x' because for some things notations like %x.%x.%x are used not
+only in Qemu. Also dumping raw data bytes with '0x' is less readable.
+
+8.2 '#' printf flag
+
+Do not use printf flag '#', like '%#x'.
+
+Rationale: there are two ways to add a '0x' prefix to printed number: '0x%...'
+and '%#...'. For consistency the only one way should be used. Arguments for
+'0x%' are:
+ - it is more popular
+ - '%#' omits the 0x for the value 0 which makes output inconsistent
@@ -1,641 +0,0 @@
-=================
-QEMU Coding Style
-=================
-
-.. contents:: Table of Contents
-
-Please use the script checkpatch.pl in the scripts directory to check
-patches before submitting.
-
-Formatting and style
-********************
-
-Whitespace
-==========
-
-Of course, the most important aspect in any coding style is whitespace.
-Crusty old coders who have trouble spotting the glasses on their noses
-can tell the difference between a tab and eight spaces from a distance
-of approximately fifteen parsecs.  Many a flamewar has been fought and
-lost on this issue.
-
-QEMU indents are four spaces.  Tabs are never used, except in Makefiles
-where they have been irreversibly coded into the syntax.
-Spaces of course are superior to tabs because:
-
-* You have just one way to specify whitespace, not two.  Ambiguity breeds
-  mistakes.
-* The confusion surrounding 'use tabs to indent, spaces to justify' is gone.
-* Tab indents push your code to the right, making your screen seriously
-  unbalanced.
-* Tabs will be rendered incorrectly on editors who are misconfigured not
-  to use tab stops of eight positions.
-* Tabs are rendered badly in patches, causing off-by-one errors in almost
-  every line.
-* It is the QEMU coding style.
-
-Do not leave whitespace dangling off the ends of lines.
-
-Multiline Indent
----------------
-
-There are several places where indent is necessary:
-
-* if/else
-* while/for
-* function definition & call
-
-When breaking up a long line to fit within line width, we need a proper indent
-for the following lines.
-
-In case of if/else, while/for, align the secondary lines just after the
-opening parenthesis of the first.
-
-For example:
-
-.. code-block:: c
-
-    if (a == 1 &&
-        b == 2) {
-
-    while (a == 1 &&
-           b == 2) {
-
-In case of function, there are several variants:
-
-* 4 spaces indent from the beginning
-* align the secondary lines just after the opening parenthesis of the first
-
-For example:
-
-.. code-block:: c
-
-    do_something(x, y,
-        z);
-
-    do_something(x, y,
-                 z);
-
-    do_something(x, do_another(y,
-                               z));
-
-Line width
-==========
-
-Lines should be 80 characters; try not to make them longer.
-
-Sometimes it is hard to do, especially when dealing with QEMU subsystems
-that use long function or symbol names.  Even in that case, do not make
-lines much longer than 80 characters.
-
-Rationale:
-
-* Some people like to tile their 24" screens with a 6x4 matrix of 80x24
-  xterms and use vi in all of them.  The best way to punish them is to
-  let them keep doing it.
-* Code and especially patches is much more readable if limited to a sane
-  line length.  Eighty is traditional.
-* The four-space indentation makes the most common excuse ("But look
-  at all that white space on the left!") moot.
-* It is the QEMU coding style.
-
-Naming
-======
-
-Variables are lower_case_with_underscores; easy to type and read.  Structured
-type names are in CamelCase; harder to type but standing out.  Enum type
-names and function type names should also be in CamelCase.  Scalar type
-names are lower_case_with_underscores_ending_with_a_t, like the POSIX
-uint64_t and family.  Note that this last convention contradicts POSIX
-and is therefore likely to be changed.
-
-When wrapping standard library functions, use the prefix ``qemu_`` to alert
-readers that they are seeing a wrapped version; otherwise avoid this prefix.
-
-Block structure
-===============
-
-Every indented statement is braced; even if the block contains just one
-statement.  The opening brace is on the line that contains the control
-flow statement that introduces the new block; the closing brace is on the
-same line as the else keyword, or on a line by itself if there is no else
-keyword.  Example:
-
-.. code-block:: c
-
-    if (a == 5) {
-        printf("a was 5.\n");
-    } else if (a == 6) {
-        printf("a was 6.\n");
-    } else {
-        printf("a was something else entirely.\n");
-    }
-
-Note that 'else if' is considered a single statement; otherwise a long if/
-else if/else if/.../else sequence would need an indent for every else
-statement.
-
-An exception is the opening brace for a function; for reasons of tradition
-and clarity it comes on a line by itself:
-
-.. code-block:: c
-
-    void a_function(void)
-    {
-        do_something();
-    }
-
-Rationale: a consistent (except for functions...) bracing style reduces
-ambiguity and avoids needless churn when lines are added or removed.
-Furthermore, it is the QEMU coding style.
-
-Declarations
-============
-
-Mixed declarations (interleaving statements and declarations within
-blocks) are generally not allowed; declarations should be at the beginning
-of blocks.
-
-Every now and then, an exception is made for declarations inside a
-#ifdef or #ifndef block: if the code looks nicer, such declarations can
-be placed at the top of the block even if there are statements above.
-On the other hand, however, it's often best to move that #ifdef/#ifndef
-block to a separate function altogether.
-
-Conditional statements
-======================
-
-When comparing a variable for (in)equality with a constant, list the
-constant on the right, as in:
-
-.. code-block:: c
-
-    if (a == 1) {
-        /* Reads like: "If a equals 1" */
-        do_something();
-    }
-
-Rationale: Yoda conditions (as in 'if (1 == a)') are awkward to read.
-Besides, good compilers already warn users when '==' is mis-typed as '=',
-even when the constant is on the right.
-
-Comment style
-=============
-
-We use traditional C-style /``*`` ``*``/ comments and avoid // comments.
-
-Rationale: The // form is valid in C99, so this is purely a matter of
-consistency of style. The checkpatch script will warn you about this.
-
-Multiline comment blocks should have a row of stars on the left,
-and the initial /``*`` and terminating ``*``/ both on their own lines:
-
-.. code-block:: c
-
-    /*
-     * like
-     * this
-     */
-
-This is the same format required by the Linux kernel coding style.
-
-(Some of the existing comments in the codebase use the GNU Coding
-Standards form which does not have stars on the left, or other
-variations; avoid these when writing new comments, but don't worry
-about converting to the preferred form unless you're editing that
-comment anyway.)
-
-Rationale: Consistency, and ease of visually picking out a multiline
-comment from the surrounding code.
-
-Language usage
-**************
-
-Preprocessor
-============
-
-Variadic macros
---------------
-
-For variadic macros, stick with this C99-like syntax:
-
-.. code-block:: c
-
-    #define DPRINTF(fmt, ...)                                       \
-        do { printf("IRQ: " fmt, ## __VA_ARGS__); } while (0)
-
-Include directives
------------------
-
-Order include directives as follows:
-
-.. code-block:: c
-
-    #include "qemu/osdep.h"  /* Always first... */
-    #include <...>           /* then system headers... */
-    #include "..."           /* and finally QEMU headers. */
-
-The "qemu/osdep.h" header contains preprocessor macros that affect the behavior
-of core system headers like <stdint.h>.  It must be the first include so that
-core system headers included by external libraries get the preprocessor macros
-that QEMU depends on.
-
-Do not include "qemu/osdep.h" from header files since the .c file will have
-already included it.
-
-C types
-=======
-
-It should be common sense to use the right type, but we have collected
-a few useful guidelines here.
-
-Scalars
-------
-
-If you're using "int" or "long", odds are good that there's a better type.
-If a variable is counting something, it should be declared with an
-unsigned type.
-
-If it's host memory-size related, size_t should be a good choice (use
-ssize_t only if required). Guest RAM memory offsets must use ram_addr_t,
-but only for RAM, it may not cover whole guest address space.
-
-If it's file-size related, use off_t.
-If it's file-offset related (i.e., signed), use off_t.
-If it's just counting small numbers use "unsigned int";
-(on all but oddball embedded systems, you can assume that that
-type is at least four bytes wide).
-
-In the event that you require a specific width, use a standard type
-like int32_t, uint32_t, uint64_t, etc.  The specific types are
-mandatory for VMState fields.
-
-Don't use Linux kernel internal types like u32, __u32 or __le32.
-
-Use hwaddr for guest physical addresses except pcibus_t
-for PCI addresses.  In addition, ram_addr_t is a QEMU internal address
-space that maps guest RAM physical addresses into an intermediate
-address space that can map to host virtual address spaces.  Generally
-speaking, the size of guest memory can always fit into ram_addr_t but
-it would not be correct to store an actual guest physical address in a
-ram_addr_t.
-
-For CPU virtual addresses there are several possible types.
-vaddr is the best type to use to hold a CPU virtual address in
-target-independent code. It is guaranteed to be large enough to hold a
-virtual address for any target, and it does not change size from target
-to target. It is always unsigned.
-target_ulong is a type the size of a virtual address on the CPU; this means
-it may be 32 or 64 bits depending on which target is being built. It should
-therefore be used only in target-specific code, and in some
-performance-critical built-per-target core code such as the TLB code.
-There is also a signed version, target_long.
-abi_ulong is for the ``*``-user targets, and represents a type the size of
-'void ``*``' in that target's ABI. (This may not be the same as the size of a
-full CPU virtual address in the case of target ABIs which use 32 bit pointers
-on 64 bit CPUs, like sparc32plus.) Definitions of structures that must match
-the target's ABI must use this type for anything that on the target is defined
-to be an 'unsigned long' or a pointer type.
-There is also a signed version, abi_long.
-
-Of course, take all of the above with a grain of salt.  If you're about
-to use some system interface that requires a type like size_t, pid_t or
-off_t, use matching types for any corresponding variables.
-
-Also, if you try to use e.g., "unsigned int" as a type, and that
-conflicts with the signedness of a related variable, sometimes
-it's best just to use the *wrong* type, if "pulling the thread"
-and fixing all related variables would be too invasive.
-
-Finally, while using descriptive types is important, be careful not to
-go overboard.  If whatever you're doing causes warnings, or requires
-casts, then reconsider or ask for help.
-
-Pointers
--------
-
-Ensure that all of your pointers are "const-correct".
-Unless a pointer is used to modify the pointed-to storage,
-give it the "const" attribute.  That way, the reader knows
-up-front that this is a read-only pointer.  Perhaps more
-importantly, if we're diligent about this, when you see a non-const
-pointer, you're guaranteed that it is used to modify the storage
-it points to, or it is aliased to another pointer that is.
-
-Typedefs
--------
-
-Typedefs are used to eliminate the redundant 'struct' keyword, since type
-names have a different style than other identifiers ("CamelCase" versus
-"snake_case").  Each named struct type should have a CamelCase name and a
-corresponding typedef.
-
-Since certain C compilers choke on duplicated typedefs, you should avoid
-them and declare a typedef only in one header file.  For common types,
-you can use "include/qemu/typedefs.h" for example.  However, as a matter
-of convenience it is also perfectly fine to use forward struct
-definitions instead of typedefs in headers and function prototypes; this
-avoids problems with duplicated typedefs and reduces the need to include
-headers from other headers.
-
-Reserved namespaces in C and POSIX
----------------------------------
-
-Underscore capital, double underscore, and underscore 't' suffixes should be
-avoided.
-
-Low level memory management
-===========================
-
-Use of the malloc/free/realloc/calloc/valloc/memalign/posix_memalign
-APIs is not allowed in the QEMU codebase. Instead of these routines,
-use the GLib memory allocation routines g_malloc/g_malloc0/g_new/
-g_new0/g_realloc/g_free or QEMU's qemu_memalign/qemu_blockalign/qemu_vfree
-APIs.
-
-Please note that g_malloc will exit on allocation failure, so there
-is no need to test for failure (as you would have to with malloc).
-Calling g_malloc with a zero size is valid and will return NULL.
-
-Prefer g_new(T, n) instead of g_malloc(sizeof(T) ``*`` n) for the following
-reasons:
-
-* It catches multiplication overflowing size_t;
-* It returns T ``*`` instead of void ``*``, letting compiler catch more type errors.
-
-Declarations like
-
-.. code-block:: c
-
-    T *v = g_malloc(sizeof(*v))
-
-are acceptable, though.
-
-Memory allocated by qemu_memalign or qemu_blockalign must be freed with
-qemu_vfree, since breaking this will cause problems on Win32.
-
-String manipulation
-===================
-
-Do not use the strncpy function.  As mentioned in the man page, it does *not*
-guarantee a NULL-terminated buffer, which makes it extremely dangerous to use.
-It also zeros trailing destination bytes out to the specified length.  Instead,
-use this similar function when possible, but note its different signature:
-
-.. code-block:: c
-
-    void pstrcpy(char *dest, int dest_buf_size, const char *src)
-
-Don't use strcat because it can't check for buffer overflows, but:
-
-.. code-block:: c
-
-    char *pstrcat(char *buf, int buf_size, const char *s)
-
-The same limitation exists with sprintf and vsprintf, so use snprintf and
-vsnprintf.
-
-QEMU provides other useful string functions:
-
-.. code-block:: c
-
-    int strstart(const char *str, const char *val, const char **ptr)
-    int stristart(const char *str, const char *val, const char **ptr)
-    int qemu_strnlen(const char *s, int max_len)
-
-There are also replacement character processing macros for isxyz and toxyz,
-so instead of e.g. isalnum you should use qemu_isalnum.
-
-Because of the memory management rules, you must use g_strdup/g_strndup
-instead of plain strdup/strndup.
-
-Printf-style functions
-======================
-
-Whenever you add a new printf-style function, i.e., one with a format
-string argument and following "..." in its prototype, be sure to use
-gcc's printf attribute directive in the prototype.
-
-This makes it so gcc's -Wformat and -Wformat-security options can do
-their jobs and cross-check format strings with the number and types
-of arguments.
-
-C standard, implementation defined and undefined behaviors
-==========================================================
-
-C code in QEMU should be written to the C99 language specification. A copy
-of the final version of the C99 standard with corrigenda TC1, TC2, and TC3
-included, formatted as a draft, can be downloaded from:
-
-    `<http://www.open-std.org/jtc1/sc22/WG14/www/docs/n1256.pdf>`_
-
-The C language specification defines regions of undefined behavior and
-implementation defined behavior (to give compiler authors enough leeway to
-produce better code).  In general, code in QEMU should follow the language
-specification and avoid both undefined and implementation defined
-constructs. ("It works fine on the gcc I tested it with" is not a valid
-argument...) However there are a few areas where we allow ourselves to
-assume certain behaviors because in practice all the platforms we care about
-behave in the same way and writing strictly conformant code would be
-painful. These are:
-
-* you may assume that integers are 2s complement representation
-* you may assume that right shift of a signed integer duplicates
-  the sign bit (ie it is an arithmetic shift, not a logical shift)
-
-In addition, QEMU assumes that the compiler does not use the latitude
-given in C99 and C11 to treat aspects of signed '<<' as undefined, as
-documented in the GNU Compiler Collection manual starting at version 4.0.
-
-Automatic memory deallocation
-=============================
-
-QEMU has a mandatory dependency either the GCC or CLang compiler. As
-such it has the freedom to make use of a C language extension for
-automatically running a cleanup function when a stack variable goes
-out of scope. This can be used to simplify function cleanup paths,
-often allowing many goto jumps to be eliminated, through automatic
-free'ing of memory.
-
-The GLib2 library provides a number of functions/macros for enabling
-automatic cleanup:
-
-  `<https://developer.gnome.org/glib/stable/glib-Miscellaneous-Macros.html>`_
-
-Most notably:
-
-* g_autofree - will invoke g_free() on the variable going out of scope
-
-* g_autoptr - for structs / objects, will invoke the cleanup func created
-  by a previous use of G_DEFINE_AUTOPTR_CLEANUP_FUNC. This is
-  supported for most GLib data types and GObjects
-
-For example, instead of
-
-.. code-block:: c
-
-    int somefunc(void) {
-        int ret = -1;
-        char *foo = g_strdup_printf("foo%", "wibble");
-        GList *bar = .....
-
-        if (eek) {
-           goto cleanup;
-        }
-
-        ret = 0;
-
-      cleanup:
-        g_free(foo);
-        g_list_free(bar);
-        return ret;
-    }
-
-Using g_autofree/g_autoptr enables the code to be written as:
-
-.. code-block:: c
-
-    int somefunc(void) {
-        g_autofree char *foo = g_strdup_printf("foo%", "wibble");
-        g_autoptr (GList) bar = .....
-
-        if (eek) {
-           return -1;
-        }
-
-        return 0;
-    }
-
-While this generally results in simpler, less leak-prone code, there
-are still some caveats to beware of
-
-* Variables declared with g_auto* MUST always be initialized,
-  otherwise the cleanup function will use uninitialized stack memory
-
-* If a variable declared with g_auto* holds a value which must
-  live beyond the life of the function, that value must be saved
-  and the original variable NULL'd out. This can be simpler using
-  g_steal_pointer
-
-
-.. code-block:: c
-
-    char *somefunc(void) {
-        g_autofree char *foo = g_strdup_printf("foo%", "wibble");
-        g_autoptr (GList) bar = .....
-
-        if (eek) {
-           return NULL;
-        }
-
-        return g_steal_pointer(&foo);
-    }
-
-
-QEMU Specific Idioms
-********************
-
-Error handling and reporting
-============================
-
-Reporting errors to the human user
----------------------------------
-
-Do not use printf(), fprintf() or monitor_printf().  Instead, use
-error_report() or error_vreport() from error-report.h.  This ensures the
-error is reported in the right place (current monitor or stderr), and in
-a uniform format.
-
-Use error_printf() & friends to print additional information.
-
-error_report() prints the current location.  In certain common cases
-like command line parsing, the current location is tracked
-automatically.  To manipulate it manually, use the loc_``*``() from
-error-report.h.
-
-Propagating errors
------------------
-
-An error can't always be reported to the user right where it's detected,
-but often needs to be propagated up the call chain to a place that can
-handle it.  This can be done in various ways.
-
-The most flexible one is Error objects.  See error.h for usage
-information.
-
-Use the simplest suitable method to communicate success / failure to
-callers.  Stick to common methods: non-negative on success / -1 on
-error, non-negative / -errno, non-null / null, or Error objects.
-
-Example: when a function returns a non-null pointer on success, and it
-can fail only in one way (as far as the caller is concerned), returning
-null on failure is just fine, and certainly simpler and a lot easier on
-the eyes than propagating an Error object through an Error ``*````*`` parameter.
-
-Example: when a function's callers need to report details on failure
-only the function really knows, use Error ``*````*``, and set suitable errors.
-
-Do not report an error to the user when you're also returning an error
-for somebody else to handle.  Leave the reporting to the place that
-consumes the error returned.
-
-Handling errors
---------------
-
-Calling exit() is fine when handling configuration errors during
-startup.  It's problematic during normal operation.  In particular,
-monitor commands should never exit().
-
-Do not call exit() or abort() to handle an error that can be triggered
-by the guest (e.g., some unimplemented corner case in guest code
-translation or device emulation).  Guests should not be able to
-terminate QEMU.
-
-Note that &error_fatal is just another way to exit(1), and &error_abort
-is just another way to abort().
-
-
-trace-events style
-==================
-
-0x prefix
---------
-
-In trace-events files, use a '0x' prefix to specify hex numbers, as in:
-
-.. code-block::
-
-    some_trace(unsigned x, uint64_t y) "x 0x%x y 0x" PRIx64
-
-An exception is made for groups of numbers that are hexadecimal by
-convention and separated by the symbols '.', '/', ':', or ' ' (such as
-PCI bus id):
-
-.. code-block::
-
-    another_trace(int cssid, int ssid, int dev_num) "bus id: %x.%x.%04x"
-
-However, you can use '0x' for such groups if you want. Anyway, be sure that
-it is obvious that numbers are in hex, ex.:
-
-.. code-block::
-
-    data_dump(uint8_t c1, uint8_t c2, uint8_t c3) "bytes (in hex): %02x %02x %02x"
-
-Rationale: hex numbers are hard to read in logs when there is no 0x prefix,
-especially when (occasionally) the representation doesn't contain any letters
-and especially in one line with other decimal numbers. Number groups are allowed
-to not use '0x' because for some things notations like %x.%x.%x are used not
-only in Qemu. Also dumping raw data bytes with '0x' is less readable.
-
-'#' printf flag
---------------
-
-Do not use printf flag '#', like '%#x'.
-
-Rationale: there are two ways to add a '0x' prefix to printed number: '0x%...'
-and '%#...'. For consistency the only one way should be used. Arguments for
-'0x%' are:
-
-* it is more popular
-* '%#' omits the 0x for the value 0 which makes output inconsistent
@@ -0,0 +1,245 @@
+1. Preprocessor
+
+1.1. Variadic macros
+
+For variadic macros, stick with this C99-like syntax:
+
+#define DPRINTF(fmt, ...)                                       \
+    do { printf("IRQ: " fmt, ## __VA_ARGS__); } while (0)
+
+1.2. Include directives
+
+Order include directives as follows:
+
+#include "qemu/osdep.h"  /* Always first... */
+#include <...>           /* then system headers... */
+#include "..."           /* and finally QEMU headers. */
+
+The "qemu/osdep.h" header contains preprocessor macros that affect the behavior
+of core system headers like <stdint.h>.  It must be the first include so that
+core system headers included by external libraries get the preprocessor macros
+that QEMU depends on.
+
+Do not include "qemu/osdep.h" from header files since the .c file will have
+already included it.
+
+2. C types
+
+It should be common sense to use the right type, but we have collected
+a few useful guidelines here.
+
+2.1. Scalars
+
+If you're using "int" or "long", odds are good that there's a better type.
+If a variable is counting something, it should be declared with an
+unsigned type.
+
+If it's host memory-size related, size_t should be a good choice (use
+ssize_t only if required). Guest RAM memory offsets must use ram_addr_t,
+but only for RAM, it may not cover whole guest address space.
+
+If it's file-size related, use off_t.
+If it's file-offset related (i.e., signed), use off_t.
+If it's just counting small numbers use "unsigned int";
+(on all but oddball embedded systems, you can assume that that
+type is at least four bytes wide).
+
+In the event that you require a specific width, use a standard type
+like int32_t, uint32_t, uint64_t, etc.  The specific types are
+mandatory for VMState fields.
+
+Don't use Linux kernel internal types like u32, __u32 or __le32.
+
+Use hwaddr for guest physical addresses except pcibus_t
+for PCI addresses.  In addition, ram_addr_t is a QEMU internal address
+space that maps guest RAM physical addresses into an intermediate
+address space that can map to host virtual address spaces.  Generally
+speaking, the size of guest memory can always fit into ram_addr_t but
+it would not be correct to store an actual guest physical address in a
+ram_addr_t.
+
+For CPU virtual addresses there are several possible types.
+vaddr is the best type to use to hold a CPU virtual address in
+target-independent code. It is guaranteed to be large enough to hold a
+virtual address for any target, and it does not change size from target
+to target. It is always unsigned.
+target_ulong is a type the size of a virtual address on the CPU; this means
+it may be 32 or 64 bits depending on which target is being built. It should
+therefore be used only in target-specific code, and in some
+performance-critical built-per-target core code such as the TLB code.
+There is also a signed version, target_long.
+abi_ulong is for the *-user targets, and represents a type the size of
+'void *' in that target's ABI. (This may not be the same as the size of a
+full CPU virtual address in the case of target ABIs which use 32 bit pointers
+on 64 bit CPUs, like sparc32plus.) Definitions of structures that must match
+the target's ABI must use this type for anything that on the target is defined
+to be an 'unsigned long' or a pointer type.
+There is also a signed version, abi_long.
+
+Of course, take all of the above with a grain of salt.  If you're about
+to use some system interface that requires a type like size_t, pid_t or
+off_t, use matching types for any corresponding variables.
+
+Also, if you try to use e.g., "unsigned int" as a type, and that
+conflicts with the signedness of a related variable, sometimes
+it's best just to use the *wrong* type, if "pulling the thread"
+and fixing all related variables would be too invasive.
+
+Finally, while using descriptive types is important, be careful not to
+go overboard.  If whatever you're doing causes warnings, or requires
+casts, then reconsider or ask for help.
+
+2.2. Pointers
+
+Ensure that all of your pointers are "const-correct".
+Unless a pointer is used to modify the pointed-to storage,
+give it the "const" attribute.  That way, the reader knows
+up-front that this is a read-only pointer.  Perhaps more
+importantly, if we're diligent about this, when you see a non-const
+pointer, you're guaranteed that it is used to modify the storage
+it points to, or it is aliased to another pointer that is.
+
+2.3. Typedefs
+Typedefs are used to eliminate the redundant 'struct' keyword.
+
+2.4. Reserved namespaces in C and POSIX
+Underscore capital, double underscore, and underscore 't' suffixes should be
+avoided.
+
+3. Low level memory management
+
+Use of the malloc/free/realloc/calloc/valloc/memalign/posix_memalign
+APIs is not allowed in the QEMU codebase. Instead of these routines,
+use the GLib memory allocation routines g_malloc/g_malloc0/g_new/
+g_new0/g_realloc/g_free or QEMU's qemu_memalign/qemu_blockalign/qemu_vfree
+APIs.
+
+Please note that g_malloc will exit on allocation failure, so there
+is no need to test for failure (as you would have to with malloc).
+Calling g_malloc with a zero size is valid and will return NULL.
+
+Prefer g_new(T, n) instead of g_malloc(sizeof(T) * n) for the following
+reasons:
+
+  a. It catches multiplication overflowing size_t;
+  b. It returns T * instead of void *, letting compiler catch more type
+     errors.
+
+Declarations like T *v = g_malloc(sizeof(*v)) are acceptable, though.
+
+Memory allocated by qemu_memalign or qemu_blockalign must be freed with
+qemu_vfree, since breaking this will cause problems on Win32.
+
+4. String manipulation
+
+Do not use the strncpy function.  As mentioned in the man page, it does *not*
+guarantee a NULL-terminated buffer, which makes it extremely dangerous to use.
+It also zeros trailing destination bytes out to the specified length.  Instead,
+use this similar function when possible, but note its different signature:
+void pstrcpy(char *dest, int dest_buf_size, const char *src)
+
+Don't use strcat because it can't check for buffer overflows, but:
+char *pstrcat(char *buf, int buf_size, const char *s)
+
+The same limitation exists with sprintf and vsprintf, so use snprintf and
+vsnprintf.
+
+QEMU provides other useful string functions:
+int strstart(const char *str, const char *val, const char **ptr)
+int stristart(const char *str, const char *val, const char **ptr)
+int qemu_strnlen(const char *s, int max_len)
+
+There are also replacement character processing macros for isxyz and toxyz,
+so instead of e.g. isalnum you should use qemu_isalnum.
+
+Because of the memory management rules, you must use g_strdup/g_strndup
+instead of plain strdup/strndup.
+
+5. Printf-style functions
+
+Whenever you add a new printf-style function, i.e., one with a format
+string argument and following "..." in its prototype, be sure to use
+gcc's printf attribute directive in the prototype.
+
+This makes it so gcc's -Wformat and -Wformat-security options can do
+their jobs and cross-check format strings with the number and types
+of arguments.
+
+6. C standard, implementation defined and undefined behaviors
+
+C code in QEMU should be written to the C99 language specification. A copy
+of the final version of the C99 standard with corrigenda TC1, TC2, and TC3
+included, formatted as a draft, can be downloaded from:
+ http://www.open-std.org/jtc1/sc22/WG14/www/docs/n1256.pdf
+
+The C language specification defines regions of undefined behavior and
+implementation defined behavior (to give compiler authors enough leeway to
+produce better code).  In general, code in QEMU should follow the language
+specification and avoid both undefined and implementation defined
+constructs. ("It works fine on the gcc I tested it with" is not a valid
+argument...) However there are a few areas where we allow ourselves to
+assume certain behaviors because in practice all the platforms we care about
+behave in the same way and writing strictly conformant code would be
+painful. These are:
+ * you may assume that integers are 2s complement representation
+ * you may assume that right shift of a signed integer duplicates
+   the sign bit (ie it is an arithmetic shift, not a logical shift)
+
+In addition, QEMU assumes that the compiler does not use the latitude
+given in C99 and C11 to treat aspects of signed '<<' as undefined, as
+documented in the GNU Compiler Collection manual starting at version 4.0.
+
+7. Error handling and reporting
+
+7.1 Reporting errors to the human user
+
+Do not use printf(), fprintf() or monitor_printf().  Instead, use
+error_report() or error_vreport() from error-report.h.  This ensures the
+error is reported in the right place (current monitor or stderr), and in
+a uniform format.
+
+Use error_printf() & friends to print additional information.
+
+error_report() prints the current location.  In certain common cases
+like command line parsing, the current location is tracked
+automatically.  To manipulate it manually, use the loc_*() from
+error-report.h.
+
+7.2 Propagating errors
+
+An error can't always be reported to the user right where it's detected,
+but often needs to be propagated up the call chain to a place that can
+handle it.  This can be done in various ways.
+
+The most flexible one is Error objects.  See error.h for usage
+information.
+
+Use the simplest suitable method to communicate success / failure to
+callers.  Stick to common methods: non-negative on success / -1 on
+error, non-negative / -errno, non-null / null, or Error objects.
+
+Example: when a function returns a non-null pointer on success, and it
+can fail only in one way (as far as the caller is concerned), returning
+null on failure is just fine, and certainly simpler and a lot easier on
+the eyes than propagating an Error object through an Error ** parameter.
+
+Example: when a function's callers need to report details on failure
+only the function really knows, use Error **, and set suitable errors.
+
+Do not report an error to the user when you're also returning an error
+for somebody else to handle.  Leave the reporting to the place that
+consumes the error returned.
+
+7.3 Handling errors
+
+Calling exit() is fine when handling configuration errors during
+startup.  It's problematic during normal operation.  In particular,
+monitor commands should never exit().
+
+Do not call exit() or abort() to handle an error that can be triggered
+by the guest (e.g., some unimplemented corner case in guest code
+translation or device emulation).  Guests should not be able to
+terminate QEMU.
+
+Note that &error_fatal is just another way to exit(1), and &error_abort
+is just another way to abort().
@@ -28,7 +28,6 @@ config VHOST_USER

 config XEN
    bool
-    select FSDEV_9P if VIRTFS

 config VIRTFS
    bool
@@ -208,7 +208,7 @@ F: disas/microblaze.c
 MIPS TCG CPUs
 M: Aurelien Jarno <aurelien@aurel32.net>
 M: Aleksandar Markovic <amarkovic@wavecomp.com>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Maintained
 F: target/mips/
 F: default-configs/*mips*
@@ -259,7 +259,7 @@ F: include/hw/ppc/
 F: disas/ppc.c

 RISC-V TCG CPUs
-M: Palmer Dabbelt <palmer@dabbelt.com>
+M: Palmer Dabbelt <palmer@sifive.com>
 M: Alistair Francis <Alistair.Francis@wdc.com>
 M: Sagar Karandikar <sagark@eecs.berkeley.edu>
 M: Bastian Koppelmann <kbastian@mail.uni-paderborn.de>
@@ -363,7 +363,7 @@ F: target/arm/kvm.c

 MIPS KVM CPUs
 M: James Hogan <jhogan@kernel.org>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Maintained
 F: target/mips/kvm.c

@@ -406,7 +406,7 @@ Guest CPU Cores (Xen)
 X86 Xen CPUs
 M: Stefano Stabellini <sstabellini@kernel.org>
 M: Anthony Perard <anthony.perard@citrix.com>
-M: Paul Durrant <paul@xen.org>
+M: Paul Durrant <paul.durrant@citrix.com>
 L: xen-devel@lists.xenproject.org
 S: Supported
 F: */xen*
@@ -495,8 +495,8 @@ F: hw/intc/pl190.c
 F: hw/sd/pl181.c
 F: hw/ssi/pl022.c
 F: include/hw/ssi/pl022.h
-F: hw/rtc/pl031.c
-F: include/hw/rtc/pl031.h
+F: hw/timer/pl031.c
+F: include/hw/timer/pl031.h
 F: include/hw/arm/primecell.h
 F: hw/timer/cmsdk-apb-timer.c
 F: include/hw/timer/cmsdk-apb-timer.h
@@ -663,7 +663,7 @@ F: hw/display/blizzard.c
 F: hw/input/lm832x.c
 F: hw/input/tsc2005.c
 F: hw/misc/cbus.c
-F: hw/rtc/twl92230.c
+F: hw/timer/twl92230.c
 F: include/hw/display/blizzard.h
 F: include/hw/input/tsc2xxx.h
 F: include/hw/misc/cbus.h
@@ -910,27 +910,6 @@ F: hw/char/mcf_uart.c
 F: hw/net/mcf_fec.c
 F: include/hw/m68k/mcf*.h

-NeXTcube
-M: Thomas Huth <huth@tuxfamily.org>
-S: Odd Fixes
-F: hw/m68k/next-*.c
-F: hw/display/next-fb.c
-F: include/hw/m68k/next-cube.h
-
-q800
-M: Laurent Vivier <laurent@vivier.eu>
-S: Maintained
-F: hw/m68k/q800.c
-F: hw/misc/mac_via.c
-F: hw/nubus/*
-F: hw/display/macfb.c
-F: hw/block/swim.c
-F: hw/m68k/bootinfo.h
-F: include/hw/misc/mac_via.h
-F: include/hw/nubus/*
-F: include/hw/display/macfb.h
-F: include/hw/block/swim.h
-
 MicroBlaze Machines
 -------------------
 petalogix_s3adsp1800
@@ -948,7 +927,7 @@ MIPS Machines
 -------------
 Jazz
 M: Hervé Poussineau <hpoussin@reactos.org>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Maintained
 F: hw/mips/mips_jazz.c
 F: hw/display/jazz_led.c
@@ -956,7 +935,7 @@ F: hw/dma/rc4030.c

 Malta
 M: Aurelien Jarno <aurelien@aurel32.net>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Maintained
 F: hw/mips/mips_malta.c
 F: hw/mips/gt64xxx_pci.c
@@ -964,20 +943,20 @@ F: tests/acceptance/linux_ssh_mips_malta.py

 Mipssim
 M: Aleksandar Markovic <amarkovic@wavecomp.com>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Odd Fixes
 F: hw/mips/mips_mipssim.c
 F: hw/net/mipsnet.c

 R4000
 M: Aurelien Jarno <aurelien@aurel32.net>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Maintained
 F: hw/mips/mips_r4k.c

 Fulong 2E
 M: Aleksandar Markovic <amarkovic@wavecomp.com>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Odd Fixes
 F: hw/mips/mips_fulong2e.c
 F: hw/isa/vt82c686.c
@@ -986,7 +965,7 @@ F: include/hw/isa/vt82c686.h

 Boston
 M: Paul Burton <pburton@wavecomp.com>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Maintained
 F: hw/core/loader-fit.c
 F: hw/mips/boston.c
@@ -1078,11 +1057,10 @@ F: hw/pci-host/prep.[hc]
 F: hw/isa/i82378.c
 F: hw/isa/pc87312.c
 F: hw/dma/i82374.c
-F: hw/rtc/m48t59-isa.c
+F: hw/timer/m48t59-isa.c
 F: include/hw/isa/pc87312.h
-F: include/hw/rtc/m48t59.h
+F: include/hw/timer/m48t59.h
 F: pc-bios/ppc_rom.bin
-F: tests/acceptance/ppc_prep_40p.py

 sPAPR
 M: David Gibson <david@gibson.dropbear.id.au>
@@ -1092,6 +1070,8 @@ F: hw/*/spapr*
 F: include/hw/*/spapr*
 F: hw/*/xics*
 F: include/hw/*/xics*
+F: pc-bios/spapr-rtas/*
+F: pc-bios/spapr-rtas.bin
 F: pc-bios/slof.bin
 F: docs/specs/ppc-spapr-hcalls.txt
 F: docs/specs/ppc-spapr-hotplug.txt
@@ -1126,7 +1106,7 @@ F: hw/ppc/sam460ex.c
 F: hw/ppc/ppc440_pcix.c
 F: hw/display/sm501*
 F: hw/ide/sii3112.c
-F: hw/rtc/m41t80.c
+F: hw/timer/m41t80.c
 F: pc-bios/canyonlands.dt[sb]
 F: pc-bios/u-boot-sam460ex-20100605.bin
 F: roms/u-boot-sam460ex
@@ -1178,8 +1158,8 @@ Sun4v
 M: Artyom Tarasenko <atar4qemu@gmail.com>
 S: Maintained
 F: hw/sparc64/niagara.c
-F: hw/rtc/sun4v-rtc.c
-F: include/hw/rtc/sun4v-rtc.h
+F: hw/timer/sun4v-rtc.c
+F: include/hw/timer/sun4v-rtc.h

 Leon3
 M: Fabien Chouteau <chouteau@adacore.com>
@@ -1188,7 +1168,6 @@ S: Maintained
 F: hw/sparc/leon3.c
 F: hw/*/grlib*
 F: include/hw/*/grlib*
-F: tests/acceptance/machine_sparc_leon3.py

 S390 Machines
 -------------
@@ -1220,7 +1199,7 @@ T: git https://github.com/borntraeger/qemu.git s390-next
 L: qemu-s390x@nongnu.org

 S390 PCI
-M: Matthew Rosato <mjrosato@linux.ibm.com>
+M: Collin Walling <walling@linux.ibm.com>
 S: Supported
 F: hw/s390x/s390-pci*
 L: qemu-s390x@nongnu.org
@@ -1241,19 +1220,18 @@ M: Marcel Apfelbaum <marcel.apfelbaum@gmail.com>
 S: Supported
 F: include/hw/i386/
 F: hw/i386/
-F: hw/pci-host/i440fx.c
+F: hw/pci-host/piix.c
 F: hw/pci-host/q35.c
 F: hw/pci-host/pam.c
-F: include/hw/pci-host/i440fx.h
 F: include/hw/pci-host/q35.h
 F: include/hw/pci-host/pam.h
-F: hw/isa/piix3.c
+F: hw/isa/piix4.c
 F: hw/isa/lpc_ich9.c
 F: hw/i2c/smbus_ich9.c
 F: hw/acpi/piix4.c
 F: hw/acpi/ich9.c
 F: include/hw/acpi/ich9.h
-F: include/hw/southbridge/piix.h
+F: include/hw/acpi/piix4.h
 F: hw/misc/sga.c
 F: hw/isa/apm.c
 F: include/hw/isa/apm.h
@@ -1278,7 +1256,7 @@ F: hw/misc/debugexit.c
 F: hw/misc/pc-testdev.c
 F: hw/timer/hpet*
 F: hw/timer/i8254*
-F: hw/rtc/mc146818rtc*
+F: hw/timer/mc146818rtc*
 F: hw/watchdog/wdt_ib700.c
 F: hw/watchdog/wdt_i6300esb.c
 F: include/hw/display/vga.h
@@ -1290,22 +1268,12 @@ F: include/hw/isa/i8259_internal.h
 F: include/hw/isa/superio.h
 F: include/hw/timer/hpet.h
 F: include/hw/timer/i8254*
-F: include/hw/rtc/mc146818rtc*
-
-microvm
-M: Sergio Lopez <slp@redhat.com>
-M: Paolo Bonzini <pbonzini@redhat.com>
-S: Maintained
-F: docs/microvm.rst
-F: hw/i386/microvm.c
-F: include/hw/i386/microvm.h
-F: pc-bios/bios-microvm.bin
+F: include/hw/timer/mc146818rtc*

 Machine core
 M: Eduardo Habkost <ehabkost@redhat.com>
 M: Marcel Apfelbaum <marcel.apfelbaum@gmail.com>
 S: Supported
-F: hw/core/cpu.c
 F: hw/core/machine-qmp-cmds.c
 F: hw/core/machine.c
 F: hw/core/null-machine.c
@@ -1313,9 +1281,10 @@ F: hw/core/numa.c
 F: hw/cpu/cluster.c
 F: qapi/machine.json
 F: qapi/machine-target.json
+F: qom/cpu.c
 F: include/hw/boards.h
-F: include/hw/core/cpu.h
 F: include/hw/cpu/cluster.h
+F: include/qom/cpu.h
 F: include/sysemu/numa.h
 T: git https://github.com/ehabkost/qemu.git machine-next

@@ -1326,11 +1295,6 @@ M: Max Filippov <jcmvbkbc@gmail.com>
 S: Maintained
 F: hw/xtensa/sim.c

-virt
-M: Max Filippov <jcmvbkbc@gmail.com>
-S: Maintained
-F: hw/xtensa/virt.c
-
 XTFPGA (LX60, LX200, ML605, KC705)
 M: Max Filippov <jcmvbkbc@gmail.com>
 S: Maintained
@@ -1438,7 +1402,6 @@ S: Odd Fixes
 F: hw/net/
 F: include/hw/net/
 F: tests/virtio-net-test.c
-F: docs/virtio-net-failover.rst
 T: git https://github.com/jasowang/qemu.git net

 Parallel NOR Flash devices
@@ -1549,7 +1512,7 @@ F: tests/virtio-balloon-test.c

 virtio-9p
 M: Greg Kurz <groug@kaod.org>
-S: Odd Fixes
+S: Supported
 F: hw/9pfs/
 X: hw/9pfs/xen-9p*
 F: fsdev/
@@ -1584,8 +1547,7 @@ F: include/hw/virtio/virtio-input.h
 F: contrib/vhost-user-input/*

 virtio-serial
-M: Laurent Vivier <lvivier@redhat.com>
-R: Amit Shah <amit@kernel.org>
+M: Amit Shah <amit@kernel.org>
 S: Supported
 F: hw/char/virtio-serial-bus.c
 F: hw/char/virtio-console.c
@@ -1594,8 +1556,7 @@ F: tests/virtio-console-test.c
 F: tests/virtio-serial-test.c

 virtio-rng
-M: Laurent Vivier <lvivier@redhat.com>
-R: Amit Shah <amit@kernel.org>
+M: Amit Shah <amit@kernel.org>
 S: Supported
 F: hw/virtio/virtio-rng.c
 F: include/hw/virtio/virtio-rng.h
@@ -1670,12 +1631,6 @@ M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: hw/net/eepro100.c

-tulip
-M: Sven Schnelle <svens@stackframe.org>
-S: Maintained
-F: hw/net/tulip.c
-F: hw/net/tulip.h
-
 Generic Loader
 M: Alistair Francis <alistair@alistair23.me>
 S: Maintained
@@ -1754,13 +1709,6 @@ F: hw/display/edid*
 F: include/hw/display/edid.h
 F: qemu-edid.c

-PIIX4 South Bridge (i82371AB)
-M: Hervé Poussineau <hpoussin@reactos.org>
-M: Philippe Mathieu-Daudé <f4bug@amsat.org>
-S: Maintained
-F: hw/isa/piix4.c
-F: include/hw/southbridge/piix.h
-
 Firmware configuration (fw_cfg)
 M: Philippe Mathieu-Daudé <philmd@redhat.com>
 R: Laszlo Ersek <lersek@redhat.com>
@@ -1861,8 +1809,8 @@ F: qapi/transaction.json
 T: git https://repo.or.cz/qemu/armbru.git block-next

 Dirty Bitmaps
+M: Fam Zheng <fam@euphon.net>
 M: John Snow <jsnow@redhat.com>
-R: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 L: qemu-block@nongnu.org
 S: Supported
 F: util/hbitmap.c
@@ -1871,6 +1819,7 @@ F: include/qemu/hbitmap.h
 F: include/block/dirty-bitmap.h
 F: tests/test-hbitmap.c
 F: docs/interop/bitmaps.rst
+T: git https://github.com/famz/qemu.git bitmaps
 T: git https://github.com/jnsnow/qemu.git bitmaps

 Character device backends
@@ -1925,7 +1874,6 @@ M: Markus Armbruster <armbru@redhat.com>
 S: Supported
 F: include/qapi/error.h
 F: include/qemu/error-report.h
-F: qapi/error.json
 F: util/error.c
 F: util/qemu-error.c

@@ -1941,7 +1889,6 @@ M: Paolo Bonzini <pbonzini@redhat.com>
 S: Supported
 F: include/exec/ioport.h
 F: ioport.c
-F: include/exec/memop.h
 F: include/exec/memory.h
 F: include/exec/ram_addr.h
 F: memory.c
@@ -1976,8 +1923,6 @@ Main loop
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Maintained
 F: cpus.c
-F: include/qemu/main-loop.h
-F: include/sysemu/runstate.h
 F: util/main-loop.c
 F: util/qemu-timer.c
 F: vl.c
@@ -2032,7 +1977,7 @@ Python scripts
 M: Eduardo Habkost <ehabkost@redhat.com>
 M: Cleber Rosa <crosa@redhat.com>
 S: Odd fixes
-F: python/qemu/*py
+F: scripts/qmp/*
 F: scripts/*.py
 F: tests/*.py

@@ -2084,7 +2029,7 @@ QEMU Guest Agent
 M: Michael Roth <mdroth@linux.vnet.ibm.com>
 S: Maintained
 F: qga/
-F: docs/interop/qemu-ga.rst
+F: qemu-ga.texi
 F: scripts/qemu-guest-agent/
 F: tests/test-qga.c
 F: docs/interop/qemu-ga-ref.texi
@@ -2100,10 +2045,12 @@ F: hw/core/qdev*
 F: include/hw/qdev*
 F: include/monitor/qdev.h
 F: include/qom/
+X: include/qom/cpu.h
 F: qapi/qom.json
 F: qapi/qdev.json
 F: qdev-monitor.c
 F: qom/
+X: qom/cpu.c
 F: tests/check-qom-interface.c
 F: tests/check-qom-proplist.c
 F: tests/test-qdev-global-props.c
@@ -2115,7 +2062,6 @@ F: monitor/monitor-internal.h
 F: monitor/qmp*
 F: monitor/misc.c
 F: monitor/monitor.c
-F: qapi/error.json
 F: docs/devel/*qmp-*
 F: docs/interop/*qmp-*
 F: scripts/qmp/
@@ -2130,7 +2076,7 @@ R: Paolo Bonzini <pbonzini@redhat.com>
 S: Maintained
 F: qtest.c
 F: accel/qtest.c
-F: tests/libqtest*
+F: tests/libqtest.*
 F: tests/libqos/
 F: tests/*-test.c

@@ -2368,12 +2314,6 @@ M: Richard Henderson <rth@twiddle.net>
 S: Maintained
 F: tcg/

-TCG Plugins
-M: Alex Bennée <alex.bennee@linaro.org>
-S: Maintained
-F: plugins/
-F: tests/plugin
-
 AArch64 TCG target
 M: Claudio Fontana <claudio.fontana@huawei.com>
 M: Claudio Fontana <claudio.fontana@gmail.com>
@@ -2398,7 +2338,7 @@ F: disas/i386.c

 MIPS TCG target
 M: Aurelien Jarno <aurelien@aurel32.net>
-R: Aleksandar Rikalo <aleksandar.rikalo@rt-rk.com>
+R: Aleksandar Rikalo <arikalo@wavecomp.com>
 S: Maintained
 F: tcg/mips/

@@ -2409,7 +2349,7 @@ F: tcg/ppc/
 F: disas/ppc.c

 RISC-V TCG target
-M: Palmer Dabbelt <palmer@dabbelt.com>
+M: Palmer Dabbelt <palmer@sifive.com>
 M: Alistair Francis <Alistair.Francis@wdc.com>
 L: qemu-riscv@nongnu.org
 S: Maintained
@@ -74,12 +74,6 @@ CONFIG_ALL=y
 config-host.mak: $(SRC_PATH)/configure $(SRC_PATH)/pc-bios $(SRC_PATH)/VERSION
 	@echo $@ is out-of-date, running configure
 	@./config.status
-
-# Force configure to re-run if the API symbols are updated
-ifeq ($(CONFIG_PLUGIN),y)
-config-host.mak: $(SRC_PATH)/plugins/qemu-plugins.symbols
-endif
-
 else
 config-host.mak:
 ifneq ($(filter-out $(UNCHECKED_GOALS),$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
@@ -90,7 +84,8 @@ endif

 include $(SRC_PATH)/rules.mak

-# lor is defined in rules.mak
+# notempy and lor are defined in rules.mak
+CONFIG_TOOLS := $(call notempty,$(TOOLS))
 CONFIG_BLOCK := $(call lor,$(CONFIG_SOFTMMU),$(CONFIG_TOOLS))

 # Create QEMU_PKGVERSION and FULL_VERSION strings
@@ -330,19 +325,8 @@ endif
 endif
 endif

-# Sphinx does not allow building manuals into the same directory as
-# the source files, so if we're doing an in-tree QEMU build we must
-# build the manuals into a subdirectory (and then install them from
-# there for 'make install'). For an out-of-tree build we can just
-# use the docs/ subdirectory in the build tree as normal.
-ifeq ($(realpath $(SRC_PATH)),$(realpath .))
-MANUAL_BUILDDIR := docs/built
-else
-MANUAL_BUILDDIR := docs
-endif
-
 ifdef BUILD_DOCS
-DOCS=qemu-doc.html qemu-doc.txt qemu.1 qemu-img.1 qemu-nbd.8 $(MANUAL_BUILDDIR)/interop/qemu-ga.8
+DOCS=qemu-doc.html qemu-doc.txt qemu.1 qemu-img.1 qemu-nbd.8 qemu-ga.8
 DOCS+=docs/interop/qemu-qmp-ref.html docs/interop/qemu-qmp-ref.txt docs/interop/qemu-qmp-ref.7
 DOCS+=docs/interop/qemu-ga-ref.html docs/interop/qemu-ga-ref.txt docs/interop/qemu-ga-ref.7
 DOCS+=docs/qemu-block-drivers.7
@@ -390,8 +374,7 @@ MINIKCONF_ARGS = \
    CONFIG_LINUX=$(CONFIG_LINUX) \
    CONFIG_PVRDMA=$(CONFIG_PVRDMA)

-MINIKCONF_INPUTS = $(SRC_PATH)/Kconfig.host $(SRC_PATH)/hw/Kconfig \
-                   $(wildcard $(SRC_PATH)/hw/*/Kconfig)
+MINIKCONF_INPUTS = $(SRC_PATH)/Kconfig.host $(SRC_PATH)/hw/Kconfig
 MINIKCONF = $(PYTHON) $(SRC_PATH)/scripts/minikconf.py \

 $(SUBDIR_DEVICES_MAK): %/config-devices.mak: default-configs/%.mak $(MINIKCONF_INPUTS) $(BUILD_DIR)/config-host.mak
@@ -517,11 +500,7 @@ capstone/all: .git-submodule-status

 .PHONY: slirp/all
 slirp/all: .git-submodule-status
-	$(call quiet-command,$(MAKE) -C $(SRC_PATH)/slirp		\
-		BUILD_DIR="$(BUILD_DIR)/slirp" 			\
-		PKG_CONFIG="$(PKG_CONFIG)" 				\
-		CC="$(CC)" AR="$(AR)" 	LD="$(LD)" RANLIB="$(RANLIB)"	\
-		CFLAGS="$(QEMU_CFLAGS) $(CFLAGS)" LDFLAGS="$(LDFLAGS)")
+	$(call quiet-command,$(MAKE) -C $(SRC_PATH)/slirp BUILD_DIR="$(BUILD_DIR)/slirp" CC="$(CC)" AR="$(AR)" LD="$(LD)" RANLIB="$(RANLIB)" CFLAGS="$(QEMU_CFLAGS) $(CFLAGS)" LDFLAGS="$(LDFLAGS)")

 # Compatibility gunk to keep make working across the rename of targets
 # for recursion, to be removed some time after 4.1.
@@ -589,20 +568,13 @@ qemu-ga$(EXESUF): QEMU_CFLAGS += -I qga/qapi-generated
 qemu-keymap$(EXESUF): LIBS += $(XKBCOMMON_LIBS)
 qemu-keymap$(EXESUF): QEMU_CFLAGS += $(XKBCOMMON_CFLAGS)

-qapi-py = $(SRC_PATH)/scripts/qapi/__init__.py \
-$(SRC_PATH)/scripts/qapi/commands.py \
-$(SRC_PATH)/scripts/qapi/common.py \
-$(SRC_PATH)/scripts/qapi/doc.py \
-$(SRC_PATH)/scripts/qapi/error.py \
+qapi-py = $(SRC_PATH)/scripts/qapi/commands.py \
 $(SRC_PATH)/scripts/qapi/events.py \
-$(SRC_PATH)/scripts/qapi/expr.py \
-$(SRC_PATH)/scripts/qapi/gen.py \
 $(SRC_PATH)/scripts/qapi/introspect.py \
-$(SRC_PATH)/scripts/qapi/parser.py \
-$(SRC_PATH)/scripts/qapi/schema.py \
-$(SRC_PATH)/scripts/qapi/source.py \
 $(SRC_PATH)/scripts/qapi/types.py \
 $(SRC_PATH)/scripts/qapi/visit.py \
+$(SRC_PATH)/scripts/qapi/common.py \
+$(SRC_PATH)/scripts/qapi/doc.py \
 $(SRC_PATH)/scripts/qapi-gen.py

 qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h \
@@ -709,7 +681,7 @@ clean: recurse-clean
 		! -path ./roms/edk2/BaseTools/Source/Python/UPT/Dll/sqlite3.dll \
 		-exec rm {} +
 	rm -f $(edk2-decompressed)
-	rm -f $(filter-out %.tlb,$(TOOLS)) $(HELPERS-y) TAGS cscope.* *.pod *~ */*~
+	rm -f $(filter-out %.tlb,$(TOOLS)) $(HELPERS-y) qemu-ga$(EXESUF) TAGS cscope.* *.pod *~ */*~
 	rm -f fsdev/*.pod scsi/*.pod
 	rm -f qemu-img-cmds.h
 	rm -f ui/shader/*-vert.h ui/shader/*-frag.h
@@ -728,6 +700,17 @@ dist: qemu-$(VERSION).tar.bz2
 qemu-%.tar.bz2:
 	$(SRC_PATH)/scripts/make-release "$(SRC_PATH)" "$(patsubst qemu-%.tar.bz2,%,$@)"

+# Sphinx does not allow building manuals into the same directory as
+# the source files, so if we're doing an in-tree QEMU build we must
+# build the manuals into a subdirectory (and then install them from
+# there for 'make install'). For an out-of-tree build we can just
+# use the docs/ subdirectory in the build tree as normal.
+ifeq ($(realpath $(SRC_PATH)),$(realpath .))
+MANUAL_BUILDDIR := docs/built
+else
+MANUAL_BUILDDIR := docs
+endif
+
 define clean-manual =
 rm -rf $(MANUAL_BUILDDIR)/$1/_static
 rm -f $(MANUAL_BUILDDIR)/$1/objects.inv $(MANUAL_BUILDDIR)/$1/searchindex.js $(MANUAL_BUILDDIR)/$1/*.html
@@ -735,7 +718,6 @@ endef

 distclean: clean
 	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi qemu-monitor-info.texi
-	rm -f tests/tcg/config-*.mak
 	rm -f config-all-devices.mak config-all-disas.mak config.status
 	rm -f $(SUBDIR_DEVICES_MAK)
 	rm -f po/*.mo tests/qemu-iotests/common.env
@@ -744,7 +726,6 @@ distclean: clean
 	rm -f qemu-doc.fn qemu-doc.fns qemu-doc.info qemu-doc.ky qemu-doc.kys
 	rm -f qemu-doc.log qemu-doc.pdf qemu-doc.pg qemu-doc.toc qemu-doc.tp
 	rm -f qemu-doc.vr qemu-doc.txt
-	rm -f qemu-plugins-ld.symbols qemu-plugins-ld64.symbols
 	rm -f config.log
 	rm -f linux-headers/asm
 	rm -f docs/version.texi
@@ -784,7 +765,7 @@ qemu-nsis.bmp \
 bamboo.dtb canyonlands.dtb petalogix-s3adsp1800.dtb petalogix-ml605.dtb \
 multiboot.bin linuxboot.bin linuxboot_dma.bin kvmvapic.bin pvh.bin \
 s390-ccw.img s390-netboot.img \
-slof.bin skiboot.lid \
+spapr-rtas.bin slof.bin skiboot.lid \
 palcode-clipper \
 u-boot.e500 u-boot-sam460-20100605.bin \
 qemu_vga.ndrv \
@@ -802,11 +783,10 @@ DESCS=
 endif

 # Note that we manually filter-out the non-Sphinx documentation which
-# is currently built into the docs/interop directory in the build tree,
-# and also any sphinx-built manpages.
+# is currently built into the docs/interop directory in the build tree.
 define install-manual =
 for d in $$(cd $(MANUAL_BUILDDIR) && find $1 -type d); do $(INSTALL_DIR) "$(DESTDIR)$(qemu_docdir)/$$d"; done
-for f in $$(cd $(MANUAL_BUILDDIR) && find $1 -type f -a '!' '(' -name '*.[0-9]' -o -name 'qemu-*-qapi.*' -o -name 'qemu-*-ref.*' ')' ); do $(INSTALL_DATA) "$(MANUAL_BUILDDIR)/$$f" "$(DESTDIR)$(qemu_docdir)/$$f"; done
+for f in $$(cd $(MANUAL_BUILDDIR) && find $1 -type f -a '!' '(' -name 'qemu-*-qapi.*' -o -name 'qemu-*-ref.*' ')' ); do $(INSTALL_DATA) "$(MANUAL_BUILDDIR)/$$f" "$(DESTDIR)$(qemu_docdir)/$$f"; done
 endef

 # Note that we deliberately do not install the "devel" manual: it is
@@ -829,7 +809,7 @@ ifdef CONFIG_POSIX
 	$(INSTALL_DATA) docs/interop/qemu-qmp-ref.7 "$(DESTDIR)$(mandir)/man7"
 	$(INSTALL_DATA) docs/qemu-block-drivers.7 "$(DESTDIR)$(mandir)/man7"
 	$(INSTALL_DATA) docs/qemu-cpu-models.7 "$(DESTDIR)$(mandir)/man7"
-ifeq ($(CONFIG_TOOLS),y)
+ifneq ($(TOOLS),)
 	$(INSTALL_DATA) qemu-img.1 "$(DESTDIR)$(mandir)/man1"
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man8"
 	$(INSTALL_DATA) qemu-nbd.8 "$(DESTDIR)$(mandir)/man8"
@@ -838,7 +818,7 @@ ifdef CONFIG_TRACE_SYSTEMTAP
 	$(INSTALL_DATA) scripts/qemu-trace-stap.1 "$(DESTDIR)$(mandir)/man1"
 endif
 ifneq (,$(findstring qemu-ga,$(TOOLS)))
-	$(INSTALL_DATA) $(MANUAL_BUILDDIR)/interop/qemu-ga.8 "$(DESTDIR)$(mandir)/man8"
+	$(INSTALL_DATA) qemu-ga.8 "$(DESTDIR)$(mandir)/man8"
 	$(INSTALL_DATA) docs/interop/qemu-ga-ref.html "$(DESTDIR)$(qemu_docdir)"
 	$(INSTALL_DATA) docs/interop/qemu-ga-ref.txt "$(DESTDIR)$(qemu_docdir)"
 	$(INSTALL_DATA) docs/interop/qemu-ga-ref.7 "$(DESTDIR)$(mandir)/man7"
@@ -861,15 +841,11 @@ endif

 ICON_SIZES=16x16 24x24 32x32 48x48 64x64 128x128 256x256 512x512

-install-includedir:
-	$(INSTALL_DIR) "$(DESTDIR)$(includedir)"
-
-install: all $(if $(BUILD_DOCS),install-doc) \
-	install-datadir install-localstatedir install-includedir \
+install: all $(if $(BUILD_DOCS),install-doc) install-datadir install-localstatedir \
 	$(if $(INSTALL_BLOBS),$(edk2-decompressed)) \
 	recurse-install
 ifneq ($(TOOLS),)
-	$(call install-prog,$(TOOLS),$(DESTDIR)$(bindir))
+	$(call install-prog,$(subst qemu-ga,qemu-ga$(EXESUF),$(TOOLS)),$(DESTDIR)$(bindir))
 endif
 ifneq ($(CONFIG_MODULES),)
 	$(INSTALL_DIR) "$(DESTDIR)$(qemu_moddir)"
@@ -927,9 +903,6 @@ endif
 		"$(DESTDIR)$(qemu_desktopdir)/qemu.desktop"
 ifdef CONFIG_GTK
 	$(MAKE) -C po $@
-endif
-ifeq ($(CONFIG_PLUGIN),y)
-	$(INSTALL_DATA) $(SRC_PATH)/include/qemu/qemu-plugin.h "$(DESTDIR)$(includedir)/qemu-plugin.h"
 endif
 	$(INSTALL_DIR) "$(DESTDIR)$(qemu_datadir)/keymaps"
 	set -e; for x in $(KEYMAPS); do \
@@ -1004,25 +977,18 @@ docs/version.texi: $(SRC_PATH)/VERSION config-host.mak
 sphinxdocs: $(MANUAL_BUILDDIR)/devel/index.html $(MANUAL_BUILDDIR)/interop/index.html $(MANUAL_BUILDDIR)/specs/index.html

 # Canned command to build a single manual
-# Arguments: $1 = manual name, $2 = Sphinx builder ('html' or 'man')
-# Note the use of different doctree for each (manual, builder) tuple;
-# this works around Sphinx not handling parallel invocation on
-# a single doctree: https://github.com/sphinx-doc/sphinx/issues/2946
-build-manual = $(call quiet-command,CONFDIR="$(qemu_confdir)" sphinx-build $(if $(V),,-q) -W -n -b $2 -D version=$(VERSION) -D release="$(FULL_VERSION)" -d .doctrees/$1-$2 $(SRC_PATH)/docs/$1 $(MANUAL_BUILDDIR)/$1 ,"SPHINX","$(MANUAL_BUILDDIR)/$1")
+build-manual = $(call quiet-command,sphinx-build $(if $(V),,-q) -W -n -b html -D version=$(VERSION) -D release="$(FULL_VERSION)" -d .doctrees/$1 $(SRC_PATH)/docs/$1 $(MANUAL_BUILDDIR)/$1 ,"SPHINX","$(MANUAL_BUILDDIR)/$1")
 # We assume all RST files in the manual's directory are used in it
 manual-deps = $(wildcard $(SRC_PATH)/docs/$1/*.rst) $(SRC_PATH)/docs/$1/conf.py $(SRC_PATH)/docs/conf.py

 $(MANUAL_BUILDDIR)/devel/index.html: $(call manual-deps,devel)
-	$(call build-manual,devel,html)
+	$(call build-manual,devel)

 $(MANUAL_BUILDDIR)/interop/index.html: $(call manual-deps,interop)
-	$(call build-manual,interop,html)
+	$(call build-manual,interop)

 $(MANUAL_BUILDDIR)/specs/index.html: $(call manual-deps,specs)
-	$(call build-manual,specs,html)
-
-$(MANUAL_BUILDDIR)/interop/qemu-ga.8: $(call manual-deps,interop)
-	$(call build-manual,interop,man)
+	$(call build-manual,specs)

 qemu-options.texi: $(SRC_PATH)/qemu-options.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"GEN","$@")
@@ -1047,6 +1013,7 @@ qemu.1: qemu-option-trace.texi
 qemu-img.1: qemu-img.texi qemu-option-trace.texi qemu-img-cmds.texi
 fsdev/virtfs-proxy-helper.1: fsdev/virtfs-proxy-helper.texi
 qemu-nbd.8: qemu-nbd.texi qemu-option-trace.texi
+qemu-ga.8: qemu-ga.texi
 docs/qemu-block-drivers.7: docs/qemu-block-drivers.texi
 docs/qemu-cpu-models.7: docs/qemu-cpu-models.texi
 scripts/qemu-trace-stap.1: scripts/qemu-trace-stap.texi
@@ -1059,7 +1026,7 @@ txt: qemu-doc.txt docs/interop/qemu-qmp-ref.txt docs/interop/qemu-ga-ref.txt
 qemu-doc.html qemu-doc.info qemu-doc.pdf qemu-doc.txt: \
 	qemu-img.texi qemu-nbd.texi qemu-options.texi \
 	qemu-tech.texi qemu-option-trace.texi \
-	qemu-deprecated.texi qemu-monitor.texi qemu-img-cmds.texi \
+	qemu-deprecated.texi qemu-monitor.texi qemu-img-cmds.texi qemu-ga.texi \
 	qemu-monitor-info.texi docs/qemu-block-drivers.texi \
 	docs/qemu-cpu-models.texi docs/security.texi

@@ -1191,7 +1158,7 @@ endif
 	@echo  ''
 	@echo  'Test targets:'
 	@echo  '  check           - Run all tests (check-help for details)'
-	@echo  '  docker          - Help about targets running tests inside containers'
+	@echo  '  docker          - Help about targets running tests inside Docker containers'
 	@echo  '  vm-help         - Help about targets running tests inside VM'
 	@echo  ''
 	@echo  'Documentation targets:'
@@ -173,7 +173,6 @@ trace-events-subdirs += hw/pci-host
 trace-events-subdirs += hw/ppc
 trace-events-subdirs += hw/rdma
 trace-events-subdirs += hw/rdma/vmw
-trace-events-subdirs += hw/rtc
 trace-events-subdirs += hw/s390x
 trace-events-subdirs += hw/scsi
 trace-events-subdirs += hw/sd
@@ -204,7 +203,6 @@ trace-events-subdirs += target/riscv
 trace-events-subdirs += target/s390x
 trace-events-subdirs += target/sparc
 trace-events-subdirs += util
-trace-events-subdirs += hw/core

 trace-events-files = $(SRC_PATH)/trace-events $(trace-events-subdirs:%=$(SRC_PATH)/%/trace-events)

@@ -39,6 +39,9 @@ endif
 PROGS=$(QEMU_PROG) $(QEMU_PROGW)
 STPFILES=

+# Makefile Tests
+include $(SRC_PATH)/tests/tcg/Makefile.include
+
 config-target.h: config-target.h-timestamp
 config-target.h-timestamp: config-target.mak

@@ -107,7 +110,7 @@ obj-y += trace/

 #########################################################
 # cpu emulator library
-obj-y += exec.o exec-vary.o
+obj-y += exec.o
 obj-y += accel/
 obj-$(CONFIG_TCG) += tcg/tcg.o tcg/tcg-op.o tcg/tcg-op-vec.o tcg/tcg-op-gvec.o
 obj-$(CONFIG_TCG) += tcg/tcg-common.o tcg/optimize.o
@@ -119,8 +122,6 @@ obj-y += disas.o
 obj-$(call notempty,$(TARGET_XML_FILES)) += gdbstub-xml.o
 LIBS := $(libs_cpu) $(LIBS)

-obj-$(CONFIG_PLUGIN) += plugins/
-
 #########################################################
 # Linux user emulator target

@@ -1,6 +1,5 @@
-===========
-QEMU README
-===========
+         QEMU README
+         ===========

 QEMU is a generic and open source machine & userspace emulator and
 virtualizer.
@@ -38,9 +37,6 @@ QEMU is multi-platform software intended to be buildable on all modern
 Linux platforms, OS-X, Win32 (via the Mingw64 toolchain) and a variety
 of other UNIX targets. The simple steps to build QEMU are:

-
-.. code-block:: shell
-
  mkdir build
  cd build
  ../configure
@@ -48,9 +44,9 @@ of other UNIX targets. The simple steps to build QEMU are:

 Additional information can also be found online via the QEMU website:

-* `<https://qemu.org/Hosts/Linux>`_
-* `<https://qemu.org/Hosts/Mac>`_
-* `<https://qemu.org/Hosts/W32>`_
+  https://qemu.org/Hosts/Linux
+  https://qemu.org/Hosts/Mac
+  https://qemu.org/Hosts/W32


 Submitting patches
@@ -58,29 +54,24 @@ Submitting patches

 The QEMU source code is maintained under the GIT version control system.

-.. code-block:: shell
-
   git clone https://git.qemu.org/git/qemu.git

 When submitting patches, one common approach is to use 'git
 format-patch' and/or 'git send-email' to format & send the mail to the
 qemu-devel@nongnu.org mailing list. All patches submitted must contain
 a 'Signed-off-by' line from the author. Patches should follow the
-guidelines set out in the CODING_STYLE.rst file.
+guidelines set out in the HACKING and CODING_STYLE files.

 Additional information on submitting patches can be found online via
 the QEMU website

-* `<https://qemu.org/Contribute/SubmitAPatch>`_
-* `<https://qemu.org/Contribute/TrivialPatches>`_
+  https://qemu.org/Contribute/SubmitAPatch
+  https://qemu.org/Contribute/TrivialPatches

 The QEMU website is also maintained under source control.

-.. code-block:: shell
-
  git clone https://git.qemu.org/git/qemu-web.git
-
-* `<https://www.qemu.org/2017/02/04/the-new-qemu-website-is-up/>`_
+  https://www.qemu.org/2017/02/04/the-new-qemu-website-is-up/

 A 'git-publish' utility was created to make above process less
 cumbersome, and is highly recommended for making regular contributions,
@@ -91,12 +82,10 @@ manually for once.

 For installation instructions, please go to

-*  `<https://github.com/stefanha/git-publish>`_
+  https://github.com/stefanha/git-publish

 The workflow with 'git-publish' is:

-.. code-block:: shell
-
  $ git checkout master -b my-feature
  $ # work on new commits, add your 'Signed-off-by' lines to each
  $ git publish
@@ -106,8 +95,6 @@ back to it in the future.

 Sending v2:

-.. code-block:: shell
-
  $ git checkout my-feature # same topic branch
  $ # making changes to the commits (using 'git rebase', for example)
  $ git publish
@@ -122,7 +109,7 @@ The QEMU project uses Launchpad as its primary upstream bug tracker. Bugs
 found when running code built from QEMU git or upstream released sources
 should be reported via:

-* `<https://bugs.launchpad.net/qemu/>`_
+  https://bugs.launchpad.net/qemu/

 If using QEMU via an operating system vendor pre-built binary package, it
 is preferable to report bugs to the vendor's own bug tracker first. If
@@ -131,7 +118,7 @@ reported via launchpad.

 For additional information on bug reporting consult:

-* `<https://qemu.org/Contribute/ReportABug>`_
+  https://qemu.org/Contribute/ReportABug


 Contact
@@ -140,11 +127,13 @@ Contact
 The QEMU community can be contacted in a number of ways, with the two
 main methods being email and IRC

-* `<mailto:qemu-devel@nongnu.org>`_
-* `<https://lists.nongnu.org/mailman/listinfo/qemu-devel>`_
-* #qemu on irc.oftc.net
+ - qemu-devel@nongnu.org
+   https://lists.nongnu.org/mailman/listinfo/qemu-devel
+ - #qemu on irc.oftc.net

 Information on additional methods of contacting the community can be
 found online via the QEMU website:

-* `<https://qemu.org/Contribute/StartHere>`_
+  https://qemu.org/Contribute/StartHere
+
+-- End
@@ -1 +1 @@
-4.1.90
+4.1.1
@@ -23,20 +23,18 @@
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
 #include "qapi/error.h"
+#include "hw/hw.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "hw/s390x/adapter.h"
 #include "exec/gdbstub.h"
 #include "sysemu/kvm_int.h"
-#include "sysemu/runstate.h"
 #include "sysemu/cpus.h"
-#include "sysemu/sysemu.h"
 #include "qemu/bswap.h"
 #include "exec/memory.h"
 #include "exec/ram_addr.h"
 #include "exec/address-spaces.h"
 #include "qemu/event_notifier.h"
-#include "qemu/main-loop.h"
 #include "trace.h"
 #include "hw/irq.h"
 #include "sysemu/sev.h"
@@ -52,7 +50,7 @@
 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
 * need to use the real host PAGE_SIZE, as that's what KVM will use.
 */
-#define PAGE_SIZE qemu_real_host_page_size
+#define PAGE_SIZE getpagesize()

 //#define DEBUG_KVM

@@ -140,7 +138,6 @@ bool kvm_direct_msi_allowed;
 bool kvm_ioeventfd_any_length_allowed;
 bool kvm_msi_use_devid;
 static bool kvm_immediate_exit;
-static hwaddr kvm_max_slot_size = ~0;

 static const KVMCapabilityInfo kvm_required_capabilites[] = {
    KVM_CAP_INFO(USER_MEMORY),
@@ -438,7 +435,7 @@ static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
 static int kvm_section_update_flags(KVMMemoryListener *kml,
                                    MemoryRegionSection *section)
 {
-    hwaddr start_addr, size, slot_size;
+    hwaddr start_addr, size;
    KVMSlot *mem;
    int ret = 0;

@@ -449,19 +446,14 @@ static int kvm_section_update_flags(KVMMemoryListener *kml,

    kvm_slots_lock(kml);

-    while (size && !ret) {
-        slot_size = MIN(kvm_max_slot_size, size);
-        mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
-        if (!mem) {
-            /* We don't have a slot if we want to trap every access. */
-            goto out;
-        }
-
-        ret = kvm_slot_update_flags(kml, mem, section->mr);
-        start_addr += slot_size;
-        size -= slot_size;
+    mem = kvm_lookup_matching_slot(kml, start_addr, size);
+    if (!mem) {
+        /* We don't have a slot if we want to trap every access. */
+        goto out;
    }

+    ret = kvm_slot_update_flags(kml, mem, section->mr);
+
 out:
    kvm_slots_unlock(kml);
    return ret;
@@ -507,7 +499,7 @@ static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
 {
    ram_addr_t start = section->offset_within_region +
                       memory_region_get_ram_addr(section->mr);
-    ram_addr_t pages = int128_get64(section->size) / qemu_real_host_page_size;
+    ram_addr_t pages = int128_get64(section->size) / getpagesize();

    cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
    return 0;
@@ -533,15 +525,11 @@ static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
    struct kvm_dirty_log d = {};
    KVMSlot *mem;
    hwaddr start_addr, size;
-    hwaddr slot_size, slot_offset = 0;
    int ret = 0;

    size = kvm_align_section(section, &start_addr);
-    while (size) {
-        MemoryRegionSection subsection = *section;
-
-        slot_size = MIN(kvm_max_slot_size, size);
-        mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
+    if (size) {
+        mem = kvm_lookup_matching_slot(kml, start_addr, size);
        if (!mem) {
            /* We don't have a slot if we want to trap every access. */
            goto out;
@@ -559,11 +547,11 @@ static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
         * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
         * a hope that sizeof(long) won't become >8 any time soon.
         */
+        size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
+                     /*HOST_LONG_BITS*/ 64) / 8;
        if (!mem->dirty_bmap) {
-            hwaddr bitmap_size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
-                                        /*HOST_LONG_BITS*/ 64) / 8;
            /* Allocate on the first log_sync, once and for all */
-            mem->dirty_bmap = g_malloc0(bitmap_size);
+            mem->dirty_bmap = g_malloc0(size);
        }

        d.dirty_bitmap = mem->dirty_bmap;
@@ -574,13 +562,7 @@ static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
            goto out;
        }

-        subsection.offset_within_region += slot_offset;
-        subsection.size = int128_make64(slot_size);
-        kvm_get_dirty_pages_log_range(&subsection, d.dirty_bitmap);
-
-        slot_offset += slot_size;
-        start_addr += slot_size;
-        size -= slot_size;
+        kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
    }
 out:
    return ret;
@@ -591,22 +573,63 @@ out:
 #define KVM_CLEAR_LOG_ALIGN  (qemu_real_host_page_size << KVM_CLEAR_LOG_SHIFT)
 #define KVM_CLEAR_LOG_MASK   (-KVM_CLEAR_LOG_ALIGN)

-static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start,
-                                  uint64_t size)
+/**
+ * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range
+ *
+ * NOTE: this will be a no-op if we haven't enabled manual dirty log
+ * protection in the host kernel because in that case this operation
+ * will be done within log_sync().
+ *
+ * @kml:     the kvm memory listener
+ * @section: the memory range to clear dirty bitmap
+ */
+static int kvm_physical_log_clear(KVMMemoryListener *kml,
+                                  MemoryRegionSection *section)
 {
    KVMState *s = kvm_state;
-    uint64_t end, bmap_start, start_delta, bmap_npages;
    struct kvm_clear_dirty_log d;
+    uint64_t start, end, bmap_start, start_delta, bmap_npages, size;
    unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size;
-    int ret;
+    KVMSlot *mem = NULL;
+    int ret, i;
+
+    if (!s->manual_dirty_log_protect) {
+        /* No need to do explicit clear */
+        return 0;
+    }
+
+    start = section->offset_within_address_space;
+    size = int128_get64(section->size);
+
+    if (!size) {
+        /* Nothing more we can do... */
+        return 0;
+    }
+
+    kvm_slots_lock(kml);
+
+    /* Find any possible slot that covers the section */
+    for (i = 0; i < s->nr_slots; i++) {
+        mem = &kml->slots[i];
+        if (mem->start_addr <= start &&
+            start + size <= mem->start_addr + mem->memory_size) {
+            break;
+        }
+    }
+
+    /*
+     * We should always find one memslot until this point, otherwise
+     * there could be something wrong from the upper layer
+     */
+    assert(mem && i != s->nr_slots);

    /*
     * We need to extend either the start or the size or both to
     * satisfy the KVM interface requirement.  Firstly, do the start
     * page alignment on 64 host pages
     */
-    bmap_start = start & KVM_CLEAR_LOG_MASK;
-    start_delta = start - bmap_start;
+    bmap_start = (start - mem->start_addr) & KVM_CLEAR_LOG_MASK;
+    start_delta = start - mem->start_addr - bmap_start;
    bmap_start /= psize;

    /*
@@ -669,7 +692,7 @@ static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start,
    /* It should never overflow.  If it happens, say something */
    assert(bmap_npages <= UINT32_MAX);
    d.num_pages = bmap_npages;
-    d.slot = mem->slot | (as_id << 16);
+    d.slot = mem->slot | (kml->as_id << 16);

    if (kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d) == -1) {
        ret = -errno;
@@ -692,66 +715,6 @@ static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start,
                 size / psize);
    /* This handles the NULL case well */
    g_free(bmap_clear);
-    return ret;
-}
-
-
-/**
- * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range
- *
- * NOTE: this will be a no-op if we haven't enabled manual dirty log
- * protection in the host kernel because in that case this operation
- * will be done within log_sync().
- *
- * @kml:     the kvm memory listener
- * @section: the memory range to clear dirty bitmap
- */
-static int kvm_physical_log_clear(KVMMemoryListener *kml,
-                                  MemoryRegionSection *section)
-{
-    KVMState *s = kvm_state;
-    uint64_t start, size, offset, count;
-    KVMSlot *mem;
-    int ret = 0, i;
-
-    if (!s->manual_dirty_log_protect) {
-        /* No need to do explicit clear */
-        return ret;
-    }
-
-    start = section->offset_within_address_space;
-    size = int128_get64(section->size);
-
-    if (!size) {
-        /* Nothing more we can do... */
-        return ret;
-    }
-
-    kvm_slots_lock(kml);
-
-    for (i = 0; i < s->nr_slots; i++) {
-        mem = &kml->slots[i];
-        /* Discard slots that are empty or do not overlap the section */
-        if (!mem->memory_size ||
-            mem->start_addr > start + size - 1 ||
-            start > mem->start_addr + mem->memory_size - 1) {
-            continue;
-        }
-
-        if (start >= mem->start_addr) {
-            /* The slot starts before section or is aligned to it.  */
-            offset = start - mem->start_addr;
-            count = MIN(mem->memory_size - offset, size);
-        } else {
-            /* The slot starts after section.  */
-            offset = 0;
-            count = MIN(mem->memory_size, size - (mem->start_addr - start));
-        }
-        ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count);
-        if (ret < 0) {
-            break;
-        }
-    }

    kvm_slots_unlock(kml);

@@ -988,14 +951,6 @@ kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
    return NULL;
 }

-void kvm_set_max_memslot_size(hwaddr max_slot_size)
-{
-    g_assert(
-        ROUND_UP(max_slot_size, qemu_real_host_page_size) == max_slot_size
-    );
-    kvm_max_slot_size = max_slot_size;
-}
-
 static void kvm_set_phys_mem(KVMMemoryListener *kml,
                             MemoryRegionSection *section, bool add)
 {
@@ -1003,7 +958,7 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml,
    int err;
    MemoryRegion *mr = section->mr;
    bool writeable = !mr->readonly && !mr->rom_device;
-    hwaddr start_addr, size, slot_size;
+    hwaddr start_addr, size;
    void *ram;

    if (!memory_region_is_ram(mr)) {
@@ -1028,52 +983,41 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml,
    kvm_slots_lock(kml);

    if (!add) {
-        do {
-            slot_size = MIN(kvm_max_slot_size, size);
-            mem = kvm_lookup_matching_slot(kml, start_addr, slot_size);
-            if (!mem) {
-                goto out;
-            }
-            if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
-                kvm_physical_sync_dirty_bitmap(kml, section);
-            }
+        mem = kvm_lookup_matching_slot(kml, start_addr, size);
+        if (!mem) {
+            goto out;
+        }
+        if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
+            kvm_physical_sync_dirty_bitmap(kml, section);
+        }

-            /* unregister the slot */
-            g_free(mem->dirty_bmap);
-            mem->dirty_bmap = NULL;
-            mem->memory_size = 0;
-            mem->flags = 0;
-            err = kvm_set_user_memory_region(kml, mem, false);
-            if (err) {
-                fprintf(stderr, "%s: error unregistering slot: %s\n",
-                        __func__, strerror(-err));
-                abort();
-            }
-            start_addr += slot_size;
-            size -= slot_size;
-        } while (size);
+        /* unregister the slot */
+        g_free(mem->dirty_bmap);
+        mem->dirty_bmap = NULL;
+        mem->memory_size = 0;
+        mem->flags = 0;
+        err = kvm_set_user_memory_region(kml, mem, false);
+        if (err) {
+            fprintf(stderr, "%s: error unregistering slot: %s\n",
+                    __func__, strerror(-err));
+            abort();
+        }
        goto out;
    }

    /* register the new slot */
-    do {
-        slot_size = MIN(kvm_max_slot_size, size);
-        mem = kvm_alloc_slot(kml);
-        mem->memory_size = slot_size;
-        mem->start_addr = start_addr;
-        mem->ram = ram;
-        mem->flags = kvm_mem_flags(mr);
+    mem = kvm_alloc_slot(kml);
+    mem->memory_size = size;
+    mem->start_addr = start_addr;
+    mem->ram = ram;
+    mem->flags = kvm_mem_flags(mr);

-        err = kvm_set_user_memory_region(kml, mem, true);
-        if (err) {
-            fprintf(stderr, "%s: error registering slot: %s\n", __func__,
-                    strerror(-err));
-            abort();
-        }
-        start_addr += slot_size;
-        ram += slot_size;
-        size -= slot_size;
-    } while (size);
+    err = kvm_set_user_memory_region(kml, mem, true);
+    if (err) {
+        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
+                strerror(-err));
+        abort();
+    }

 out:
    kvm_slots_unlock(kml);
@@ -1841,7 +1785,7 @@ static int kvm_init(MachineState *ms)
     * even with KVM.  TARGET_PAGE_SIZE is assumed to be the minimum
     * page size for the system though.
     */
-    assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size);
+    assert(TARGET_PAGE_SIZE <= getpagesize());

    s->sigmask_len = 8;

@@ -2913,7 +2857,6 @@ static bool kvm_accel_has_memory(MachineState *ms, AddressSpace *as,

    for (i = 0; i < kvm->nr_as; ++i) {
        if (kvm->as[i].as == as && kvm->as[i].ml) {
-            size = MIN(kvm_max_slot_size, size);
            return NULL != kvm_lookup_matching_slot(kvm->as[i].ml,
                                                    start_addr, size);
        }
@@ -11,8 +11,10 @@
 */

 #include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "cpu.h"
 #include "tcg/tcg.h"
+#include "exec/cpu-common.h"
 #include "exec/exec-all.h"

 void tb_flush(CPUState *cpu)
@@ -6,4 +6,3 @@ obj-y += translator.o

 obj-$(CONFIG_USER_ONLY) += user-exec.o
 obj-$(call lnot,$(CONFIG_SOFTMMU)) += user-exec-stub.o
-obj-$(CONFIG_PLUGIN) += plugin-gen.o
@@ -1,54 +0,0 @@
-/*
- * Common Atomic Helper Functions
- *
- * This file should be included before the various instantiations of
- * the atomic_template.h helpers.
- *
- * Copyright (c) 2019 Linaro
- * Written by Alex Bennée <alex.bennee@linaro.org>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- */
-
-static inline
-void atomic_trace_rmw_pre(CPUArchState *env, target_ulong addr, uint16_t info)
-{
-    CPUState *cpu = env_cpu(env);
-
-    trace_guest_mem_before_exec(cpu, addr, info);
-    trace_guest_mem_before_exec(cpu, addr, info | TRACE_MEM_ST);
-}
-
-static inline void
-atomic_trace_rmw_post(CPUArchState *env, target_ulong addr, uint16_t info)
-{
-    qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info);
-    qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info | TRACE_MEM_ST);
-}
-
-static inline
-void atomic_trace_ld_pre(CPUArchState *env, target_ulong addr, uint16_t info)
-{
-    trace_guest_mem_before_exec(env_cpu(env), addr, info);
-}
-
-static inline
-void atomic_trace_ld_post(CPUArchState *env, target_ulong addr, uint16_t info)
-{
-    qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info);
-}
-
-static inline
-void atomic_trace_st_pre(CPUArchState *env, target_ulong addr, uint16_t info)
-{
-    trace_guest_mem_before_exec(env_cpu(env), addr, info);
-}
-
-static inline
-void atomic_trace_st_post(CPUArchState *env, target_ulong addr, uint16_t info)
-{
-    qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, info);
-}
@@ -18,7 +18,6 @@
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

-#include "qemu/plugin.h"
 #include "trace/mem.h"

 #if DATA_SIZE == 16
@@ -60,6 +59,26 @@
 # define ABI_TYPE  uint32_t
 #endif

+#define ATOMIC_TRACE_RMW do {                                           \
+        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
+                                                                        \
+        trace_guest_mem_before_exec(env_cpu(env), addr, info);      \
+        trace_guest_mem_before_exec(env_cpu(env), addr,             \
+                                    info | TRACE_MEM_ST);               \
+    } while (0)
+
+#define ATOMIC_TRACE_LD do {                                            \
+        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
+                                                                        \
+        trace_guest_mem_before_exec(env_cpu(env), addr, info);      \
+    } while (0)
+
+# define ATOMIC_TRACE_ST do {                                           \
+        uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
+                                                                        \
+        trace_guest_mem_before_exec(env_cpu(env), addr, info);      \
+    } while (0)
+
 /* Define host-endian atomic operations.  Note that END is used within
   the ATOMIC_NAME macro, and redefined below.  */
 #if DATA_SIZE == 1
@@ -79,17 +98,14 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
-                                                           ATOMIC_MMU_IDX);

-    atomic_trace_rmw_pre(env, addr, info);
+    ATOMIC_TRACE_RMW;
 #if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, cmpv, newv);
 #else
    ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
 #endif
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr, info);
    return ret;
 }

@@ -99,13 +115,10 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 {
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
-                                                           ATOMIC_MMU_IDX);

-    atomic_trace_ld_pre(env, addr, info);
+    ATOMIC_TRACE_LD;
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_ld_post(env, addr, info);
    return val;
 }

@@ -114,13 +127,10 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 {
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true,
-                                                          ATOMIC_MMU_IDX);

-    atomic_trace_st_pre(env, addr, info);
+    ATOMIC_TRACE_ST;
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_st_post(env, addr, info);
 }
 #endif
 #else
@@ -130,31 +140,24 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
-                                                          ATOMIC_MMU_IDX);

-    atomic_trace_rmw_pre(env, addr, info);
+    ATOMIC_TRACE_RMW;
    ret = atomic_xchg__nocheck(haddr, val);
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr, info);
    return ret;
 }

 #define GEN_ATOMIC_HELPER(X)                                        \
 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
-                        ABI_TYPE val EXTRA_ARGS)                    \
+                 ABI_TYPE val EXTRA_ARGS)                           \
 {                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
-                                                           false,   \
-                                                           ATOMIC_MMU_IDX); \
                                                                    \
-    atomic_trace_rmw_pre(env, addr, info);                          \
+    ATOMIC_TRACE_RMW;                                               \
    ret = atomic_##X(haddr, val);                                   \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr, info);                         \
    return ret;                                                     \
 }

@@ -183,11 +186,8 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE cmp, old, new, val = xval;                           \
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
-                                                           false,   \
-                                                           ATOMIC_MMU_IDX); \
                                                                    \
-    atomic_trace_rmw_pre(env, addr, info);                          \
+    ATOMIC_TRACE_RMW;                                               \
    smp_mb();                                                       \
    cmp = atomic_read__nocheck(haddr);                              \
    do {                                                            \
@@ -195,7 +195,6 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
        cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
    } while (cmp != old);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
 }

@@ -233,18 +232,14 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
-                                                           false,
-                                                           ATOMIC_MMU_IDX);

-    atomic_trace_rmw_pre(env, addr, info);
+    ATOMIC_TRACE_RMW;
 #if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
 #else
    ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 #endif
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
 }

@@ -254,14 +249,10 @@ ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 {
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
-                                                           false,
-                                                           ATOMIC_MMU_IDX);

-    atomic_trace_ld_pre(env, addr, info);
+    ATOMIC_TRACE_LD;
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_ld_post(env, addr, info);
    return BSWAP(val);
 }

@@ -270,16 +261,11 @@ void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 {
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
-                                                           true,
-                                                           ATOMIC_MMU_IDX);

-    val = BSWAP(val);
-    atomic_trace_st_pre(env, addr, info);
+    ATOMIC_TRACE_ST;
    val = BSWAP(val);
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_st_post(env, addr, info);
 }
 #endif
 #else
@@ -289,32 +275,24 @@ ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    ABI_TYPE ret;
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
-                                                           false,
-                                                           ATOMIC_MMU_IDX);

-    atomic_trace_rmw_pre(env, addr, info);
+    ATOMIC_TRACE_RMW;
    ret = atomic_xchg__nocheck(haddr, BSWAP(val));
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
 }

 #define GEN_ATOMIC_HELPER(X)                                        \
 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
-                        ABI_TYPE val EXTRA_ARGS)                    \
+                 ABI_TYPE val EXTRA_ARGS)                           \
 {                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
-                                                           false,   \
-                                                           ATOMIC_MMU_IDX); \
                                                                    \
-    atomic_trace_rmw_pre(env, addr, info);                          \
+    ATOMIC_TRACE_RMW;                                               \
    ret = atomic_##X(haddr, BSWAP(val));                            \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr, info);                         \
    return BSWAP(ret);                                              \
 }

@@ -341,11 +319,8 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
-    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
-                                                           false,   \
-                                                           ATOMIC_MMU_IDX); \
                                                                    \
-    atomic_trace_rmw_pre(env, addr, info);                          \
+    ATOMIC_TRACE_RMW;                                               \
    smp_mb();                                                       \
    ldn = atomic_read__nocheck(haddr);                              \
    do {                                                            \
@@ -353,7 +328,6 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
        ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
    } while (ldo != ldn);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
 }

@@ -381,6 +355,10 @@ GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 #undef MEND
 #endif /* DATA_SIZE > 1 */

+#undef ATOMIC_TRACE_ST
+#undef ATOMIC_TRACE_LD
+#undef ATOMIC_TRACE_RMW
+
 #undef BSWAP
 #undef ABI_TYPE
 #undef DATA_TYPE
@@ -169,6 +169,7 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
    }
 #endif /* DEBUG_DISAS */

+    cpu->can_do_io = !use_icount;
    ret = tcg_qemu_tb_exec(env, tb_ptr);
    cpu->can_do_io = 1;
    last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK);
@@ -238,6 +239,8 @@ void cpu_exec_step_atomic(CPUState *cpu)
    uint32_t flags;
    uint32_t cflags = 1;
    uint32_t cf_mask = cflags & CF_HASH_MASK;
+    /* volatile because we modify it between setjmp and longjmp */
+    volatile bool in_exclusive_region = false;

    if (sigsetjmp(cpu->jmp_env, 0) == 0) {
        tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags, cf_mask);
@@ -251,6 +254,7 @@ void cpu_exec_step_atomic(CPUState *cpu)

        /* Since we got here, we know that parallel_cpus must be true.  */
        parallel_cpus = false;
+        in_exclusive_region = true;
        cc->cpu_exec_enter(cpu);
        /* execute the generated code */
        trace_exec_tb(tb, pc);
@@ -268,10 +272,9 @@ void cpu_exec_step_atomic(CPUState *cpu)
            qemu_mutex_unlock_iothread();
        }
        assert_no_pages_locked();
-        qemu_plugin_disable_mem_helpers(cpu);
    }

-    if (cpu_in_exclusive_context(cpu)) {
+    if (in_exclusive_region) {
        /* We might longjump out of either the codegen or the
         * execution, so must make sure we only end the exclusive
         * region if we started it.
@@ -702,8 +705,6 @@ int cpu_exec(CPUState *cpu)
        if (qemu_mutex_iothread_locked()) {
            qemu_mutex_unlock_iothread();
        }
-        qemu_plugin_disable_mem_helpers(cpu);
-
        assert_no_pages_locked();
    }

@@ -1,932 +0,0 @@
-/*
- * plugin-gen.c - TCG-related bits of plugin infrastructure
- *
- * Copyright (C) 2018, Emilio G. Cota <cota@braap.org>
- * License: GNU GPL, version 2 or later.
- *   See the COPYING file in the top-level directory.
- *
- * We support instrumentation at an instruction granularity. That is,
- * if a plugin wants to instrument the memory accesses performed by a
- * particular instruction, it can just do that instead of instrumenting
- * all memory accesses. Thus, in order to do this we first have to
- * translate a TB, so that plugins can decide what/where to instrument.
- *
- * Injecting the desired instrumentation could be done with a second
- * translation pass that combined the instrumentation requests, but that
- * would be ugly and inefficient since we would decode the guest code twice.
- * Instead, during TB translation we add "empty" instrumentation calls for all
- * possible instrumentation events, and then once we collect the instrumentation
- * requests from plugins, we either "fill in" those empty events or remove them
- * if they have no requests.
- *
- * When "filling in" an event we first copy the empty callback's TCG ops. This
- * might seem unnecessary, but it is done to support an arbitrary number
- * of callbacks per event. Take for example a regular instruction callback.
- * We first generate a callback to an empty helper function. Then, if two
- * plugins register one callback each for this instruction, we make two copies
- * of the TCG ops generated for the empty callback, substituting the function
- * pointer that points to the empty helper function with the plugins' desired
- * callback functions. After that we remove the empty callback's ops.
- *
- * Note that the location in TCGOp.args[] of the pointer to a helper function
- * varies across different guest and host architectures. Instead of duplicating
- * the logic that figures this out, we rely on the fact that the empty
- * callbacks point to empty functions that are unique pointers in the program.
- * Thus, to find the right location we just have to look for a match in
- * TCGOp.args[]. This is the main reason why we first copy an empty callback's
- * TCG ops and then fill them in; regardless of whether we have one or many
- * callbacks for that event, the logic to add all of them is the same.
- *
- * When generating more than one callback per event, we make a small
- * optimization to avoid generating redundant operations. For instance, for the
- * second and all subsequent callbacks of an event, we do not need to reload the
- * CPU's index into a TCG temp, since the first callback did it already.
- */
-#include "qemu/osdep.h"
-#include "cpu.h"
-#include "tcg/tcg.h"
-#include "tcg/tcg-op.h"
-#include "trace/mem.h"
-#include "exec/exec-all.h"
-#include "exec/plugin-gen.h"
-#include "exec/translator.h"
-
-#ifdef CONFIG_SOFTMMU
-# define CONFIG_SOFTMMU_GATE 1
-#else
-# define CONFIG_SOFTMMU_GATE 0
-#endif
-
-/*
- * plugin_cb_start TCG op args[]:
- * 0: enum plugin_gen_from
- * 1: enum plugin_gen_cb
- * 2: set to 1 for mem callback that is a write, 0 otherwise.
- */
-
-enum plugin_gen_from {
-    PLUGIN_GEN_FROM_TB,
-    PLUGIN_GEN_FROM_INSN,
-    PLUGIN_GEN_FROM_MEM,
-    PLUGIN_GEN_AFTER_INSN,
-    PLUGIN_GEN_N_FROMS,
-};
-
-enum plugin_gen_cb {
-    PLUGIN_GEN_CB_UDATA,
-    PLUGIN_GEN_CB_INLINE,
-    PLUGIN_GEN_CB_MEM,
-    PLUGIN_GEN_ENABLE_MEM_HELPER,
-    PLUGIN_GEN_DISABLE_MEM_HELPER,
-    PLUGIN_GEN_N_CBS,
-};
-
-/*
- * These helpers are stubs that get dynamically switched out for calls
- * direct to the plugin if they are subscribed to.
- */
-void HELPER(plugin_vcpu_udata_cb)(uint32_t cpu_index, void *udata)
-{ }
-
-void HELPER(plugin_vcpu_mem_cb)(unsigned int vcpu_index,
-                                qemu_plugin_meminfo_t info, uint64_t vaddr,
-                                void *userdata)
-{ }
-
-static void do_gen_mem_cb(TCGv vaddr, uint32_t info)
-{
-    TCGv_i32 cpu_index = tcg_temp_new_i32();
-    TCGv_i32 meminfo = tcg_const_i32(info);
-    TCGv_i64 vaddr64 = tcg_temp_new_i64();
-    TCGv_ptr udata = tcg_const_ptr(NULL);
-
-    tcg_gen_ld_i32(cpu_index, cpu_env,
-                   -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index));
-    tcg_gen_extu_tl_i64(vaddr64, vaddr);
-
-    gen_helper_plugin_vcpu_mem_cb(cpu_index, meminfo, vaddr64, udata);
-
-    tcg_temp_free_ptr(udata);
-    tcg_temp_free_i64(vaddr64);
-    tcg_temp_free_i32(meminfo);
-    tcg_temp_free_i32(cpu_index);
-}
-
-static void gen_empty_udata_cb(void)
-{
-    TCGv_i32 cpu_index = tcg_temp_new_i32();
-    TCGv_ptr udata = tcg_const_ptr(NULL); /* will be overwritten later */
-
-    tcg_gen_ld_i32(cpu_index, cpu_env,
-                   -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index));
-    gen_helper_plugin_vcpu_udata_cb(cpu_index, udata);
-
-    tcg_temp_free_ptr(udata);
-    tcg_temp_free_i32(cpu_index);
-}
-
-/*
- * For now we only support addi_i64.
- * When we support more ops, we can generate one empty inline cb for each.
- */
-static void gen_empty_inline_cb(void)
-{
-    TCGv_i64 val = tcg_temp_new_i64();
-    TCGv_ptr ptr = tcg_const_ptr(NULL); /* overwritten later */
-
-    tcg_gen_ld_i64(val, ptr, 0);
-    /* pass an immediate != 0 so that it doesn't get optimized away */
-    tcg_gen_addi_i64(val, val, 0xdeadface);
-    tcg_gen_st_i64(val, ptr, 0);
-    tcg_temp_free_ptr(ptr);
-    tcg_temp_free_i64(val);
-}
-
-static void gen_empty_mem_cb(TCGv addr, uint32_t info)
-{
-    do_gen_mem_cb(addr, info);
-}
-
-/*
- * Share the same function for enable/disable. When enabling, the NULL
- * pointer will be overwritten later.
- */
-static void gen_empty_mem_helper(void)
-{
-    TCGv_ptr ptr;
-
-    ptr = tcg_const_ptr(NULL);
-    tcg_gen_st_ptr(ptr, cpu_env, offsetof(CPUState, plugin_mem_cbs) -
-                                 offsetof(ArchCPU, env));
-    tcg_temp_free_ptr(ptr);
-}
-
-static inline
-void gen_plugin_cb_start(enum plugin_gen_from from,
-                         enum plugin_gen_cb type, unsigned wr)
-{
-    TCGOp *op;
-
-    tcg_gen_plugin_cb_start(from, type, wr);
-    op = tcg_last_op();
-    QSIMPLEQ_INSERT_TAIL(&tcg_ctx->plugin_ops, op, plugin_link);
-}
-
-static void gen_wrapped(enum plugin_gen_from from,
-                        enum plugin_gen_cb type, void (*func)(void))
-{
-    gen_plugin_cb_start(from, type, 0);
-    func();
-    tcg_gen_plugin_cb_end();
-}
-
-static inline void plugin_gen_empty_callback(enum plugin_gen_from from)
-{
-    switch (from) {
-    case PLUGIN_GEN_AFTER_INSN:
-        gen_wrapped(from, PLUGIN_GEN_DISABLE_MEM_HELPER,
-                    gen_empty_mem_helper);
-        break;
-    case PLUGIN_GEN_FROM_INSN:
-        /*
-         * Note: plugin_gen_inject() relies on ENABLE_MEM_HELPER being
-         * the first callback of an instruction
-         */
-        gen_wrapped(from, PLUGIN_GEN_ENABLE_MEM_HELPER,
-                    gen_empty_mem_helper);
-        /* fall through */
-    case PLUGIN_GEN_FROM_TB:
-        gen_wrapped(from, PLUGIN_GEN_CB_UDATA, gen_empty_udata_cb);
-        gen_wrapped(from, PLUGIN_GEN_CB_INLINE, gen_empty_inline_cb);
-        break;
-    default:
-        g_assert_not_reached();
-    }
-}
-
-union mem_gen_fn {
-    void (*mem_fn)(TCGv, uint32_t);
-    void (*inline_fn)(void);
-};
-
-static void gen_mem_wrapped(enum plugin_gen_cb type,
-                            const union mem_gen_fn *f, TCGv addr,
-                            uint32_t info, bool is_mem)
-{
-    int wr = !!(info & TRACE_MEM_ST);
-
-    gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, type, wr);
-    if (is_mem) {
-        f->mem_fn(addr, info);
-    } else {
-        f->inline_fn();
-    }
-    tcg_gen_plugin_cb_end();
-}
-
-void plugin_gen_empty_mem_callback(TCGv addr, uint32_t info)
-{
-    union mem_gen_fn fn;
-
-    fn.mem_fn = gen_empty_mem_cb;
-    gen_mem_wrapped(PLUGIN_GEN_CB_MEM, &fn, addr, info, true);
-
-    fn.inline_fn = gen_empty_inline_cb;
-    gen_mem_wrapped(PLUGIN_GEN_CB_INLINE, &fn, 0, info, false);
-}
-
-static TCGOp *find_op(TCGOp *op, TCGOpcode opc)
-{
-    while (op) {
-        if (op->opc == opc) {
-            return op;
-        }
-        op = QTAILQ_NEXT(op, link);
-    }
-    return NULL;
-}
-
-static TCGOp *rm_ops_range(TCGOp *begin, TCGOp *end)
-{
-    TCGOp *ret = QTAILQ_NEXT(end, link);
-
-    QTAILQ_REMOVE_SEVERAL(&tcg_ctx->ops, begin, end, link);
-    return ret;
-}
-
-/* remove all ops until (and including) plugin_cb_end */
-static TCGOp *rm_ops(TCGOp *op)
-{
-    TCGOp *end_op = find_op(op, INDEX_op_plugin_cb_end);
-
-    tcg_debug_assert(end_op);
-    return rm_ops_range(op, end_op);
-}
-
-static TCGOp *copy_op_nocheck(TCGOp **begin_op, TCGOp *op)
-{
-    *begin_op = QTAILQ_NEXT(*begin_op, link);
-    tcg_debug_assert(*begin_op);
-    op = tcg_op_insert_after(tcg_ctx, op, (*begin_op)->opc);
-    memcpy(op->args, (*begin_op)->args, sizeof(op->args));
-    return op;
-}
-
-static TCGOp *copy_op(TCGOp **begin_op, TCGOp *op, TCGOpcode opc)
-{
-    op = copy_op_nocheck(begin_op, op);
-    tcg_debug_assert((*begin_op)->opc == opc);
-    return op;
-}
-
-static TCGOp *copy_extu_i32_i64(TCGOp **begin_op, TCGOp *op)
-{
-    if (TCG_TARGET_REG_BITS == 32) {
-        /* mov_i32 */
-        op = copy_op(begin_op, op, INDEX_op_mov_i32);
-        /* movi_i32 */
-        op = copy_op(begin_op, op, INDEX_op_movi_i32);
-    } else {
-        /* extu_i32_i64 */
-        op = copy_op(begin_op, op, INDEX_op_extu_i32_i64);
-    }
-    return op;
-}
-
-static TCGOp *copy_mov_i64(TCGOp **begin_op, TCGOp *op)
-{
-    if (TCG_TARGET_REG_BITS == 32) {
-        /* 2x mov_i32 */
-        op = copy_op(begin_op, op, INDEX_op_mov_i32);
-        op = copy_op(begin_op, op, INDEX_op_mov_i32);
-    } else {
-        /* mov_i64 */
-        op = copy_op(begin_op, op, INDEX_op_mov_i64);
-    }
-    return op;
-}
-
-static TCGOp *copy_movi_i64(TCGOp **begin_op, TCGOp *op, uint64_t v)
-{
-    if (TCG_TARGET_REG_BITS == 32) {
-        /* 2x movi_i32 */
-        op = copy_op(begin_op, op, INDEX_op_movi_i32);
-        op->args[1] = v;
-
-        op = copy_op(begin_op, op, INDEX_op_movi_i32);
-        op->args[1] = v >> 32;
-    } else {
-        /* movi_i64 */
-        op = copy_op(begin_op, op, INDEX_op_movi_i64);
-        op->args[1] = v;
-    }
-    return op;
-}
-
-static TCGOp *copy_const_ptr(TCGOp **begin_op, TCGOp *op, void *ptr)
-{
-    if (UINTPTR_MAX == UINT32_MAX) {
-        /* movi_i32 */
-        op = copy_op(begin_op, op, INDEX_op_movi_i32);
-        op->args[1] = (uintptr_t)ptr;
-    } else {
-        /* movi_i64 */
-        op = copy_movi_i64(begin_op, op, (uint64_t)(uintptr_t)ptr);
-    }
-    return op;
-}
-
-static TCGOp *copy_const_i64(TCGOp **begin_op, TCGOp *op, uint64_t v)
-{
-    return copy_movi_i64(begin_op, op, v);
-}
-
-static TCGOp *copy_extu_tl_i64(TCGOp **begin_op, TCGOp *op)
-{
-    if (TARGET_LONG_BITS == 32) {
-        /* extu_i32_i64 */
-        op = copy_extu_i32_i64(begin_op, op);
-    } else {
-        /* mov_i64 */
-        op = copy_mov_i64(begin_op, op);
-    }
-    return op;
-}
-
-static TCGOp *copy_ld_i64(TCGOp **begin_op, TCGOp *op)
-{
-    if (TCG_TARGET_REG_BITS == 32) {
-        /* 2x ld_i32 */
-        op = copy_op(begin_op, op, INDEX_op_ld_i32);
-        op = copy_op(begin_op, op, INDEX_op_ld_i32);
-    } else {
-        /* ld_i64 */
-        op = copy_op(begin_op, op, INDEX_op_ld_i64);
-    }
-    return op;
-}
-
-static TCGOp *copy_st_i64(TCGOp **begin_op, TCGOp *op)
-{
-    if (TCG_TARGET_REG_BITS == 32) {
-        /* 2x st_i32 */
-        op = copy_op(begin_op, op, INDEX_op_st_i32);
-        op = copy_op(begin_op, op, INDEX_op_st_i32);
-    } else {
-        /* st_i64 */
-        op = copy_op(begin_op, op, INDEX_op_st_i64);
-    }
-    return op;
-}
-
-static TCGOp *copy_add_i64(TCGOp **begin_op, TCGOp *op)
-{
-    if (TCG_TARGET_REG_BITS == 32) {
-        /* all 32-bit backends must implement add2_i32 */
-        g_assert(TCG_TARGET_HAS_add2_i32);
-        op = copy_op(begin_op, op, INDEX_op_add2_i32);
-    } else {
-        op = copy_op(begin_op, op, INDEX_op_add_i64);
-    }
-    return op;
-}
-
-static TCGOp *copy_st_ptr(TCGOp **begin_op, TCGOp *op)
-{
-    if (UINTPTR_MAX == UINT32_MAX) {
-        /* st_i32 */
-        op = copy_op(begin_op, op, INDEX_op_st_i32);
-    } else {
-        /* st_i64 */
-        op = copy_st_i64(begin_op, op);
-    }
-    return op;
-}
-
-static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *empty_func,
-                        void *func, unsigned tcg_flags, int *cb_idx)
-{
-    /* copy all ops until the call */
-    do {
-        op = copy_op_nocheck(begin_op, op);
-    } while (op->opc != INDEX_op_call);
-
-    /* fill in the op call */
-    op->param1 = (*begin_op)->param1;
-    op->param2 = (*begin_op)->param2;
-    tcg_debug_assert(op->life == 0);
-    if (*cb_idx == -1) {
-        int i;
-
-        /*
-         * Instead of working out the position of the callback in args[], just
-         * look for @empty_func, since it should be a unique pointer.
-         */
-        for (i = 0; i < MAX_OPC_PARAM_ARGS; i++) {
-            if ((uintptr_t)(*begin_op)->args[i] == (uintptr_t)empty_func) {
-                *cb_idx = i;
-                break;
-            }
-        }
-        tcg_debug_assert(i < MAX_OPC_PARAM_ARGS);
-    }
-    op->args[*cb_idx] = (uintptr_t)func;
-    op->args[*cb_idx + 1] = tcg_flags;
-
-    return op;
-}
-
-static TCGOp *append_udata_cb(const struct qemu_plugin_dyn_cb *cb,
-                              TCGOp *begin_op, TCGOp *op, int *cb_idx)
-{
-    /* const_ptr */
-    op = copy_const_ptr(&begin_op, op, cb->userp);
-
-    /* copy the ld_i32, but note that we only have to copy it once */
-    begin_op = QTAILQ_NEXT(begin_op, link);
-    tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32);
-    if (*cb_idx == -1) {
-        op = tcg_op_insert_after(tcg_ctx, op, INDEX_op_ld_i32);
-        memcpy(op->args, begin_op->args, sizeof(op->args));
-    }
-
-    /* call */
-    op = copy_call(&begin_op, op, HELPER(plugin_vcpu_udata_cb),
-                   cb->f.vcpu_udata, cb->tcg_flags, cb_idx);
-
-    return op;
-}
-
-static TCGOp *append_inline_cb(const struct qemu_plugin_dyn_cb *cb,
-                               TCGOp *begin_op, TCGOp *op,
-                               int *unused)
-{
-    /* const_ptr */
-    op = copy_const_ptr(&begin_op, op, cb->userp);
-
-    /* ld_i64 */
-    op = copy_ld_i64(&begin_op, op);
-
-    /* const_i64 */
-    op = copy_const_i64(&begin_op, op, cb->inline_insn.imm);
-
-    /* add_i64 */
-    op = copy_add_i64(&begin_op, op);
-
-    /* st_i64 */
-    op = copy_st_i64(&begin_op, op);
-
-    return op;
-}
-
-static TCGOp *append_mem_cb(const struct qemu_plugin_dyn_cb *cb,
-                            TCGOp *begin_op, TCGOp *op, int *cb_idx)
-{
-    enum plugin_gen_cb type = begin_op->args[1];
-
-    tcg_debug_assert(type == PLUGIN_GEN_CB_MEM);
-
-    /* const_i32 == movi_i32 ("info", so it remains as is) */
-    op = copy_op(&begin_op, op, INDEX_op_movi_i32);
-
-    /* const_ptr */
-    op = copy_const_ptr(&begin_op, op, cb->userp);
-
-    /* copy the ld_i32, but note that we only have to copy it once */
-    begin_op = QTAILQ_NEXT(begin_op, link);
-    tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32);
-    if (*cb_idx == -1) {
-        op = tcg_op_insert_after(tcg_ctx, op, INDEX_op_ld_i32);
-        memcpy(op->args, begin_op->args, sizeof(op->args));
-    }
-
-    /* extu_tl_i64 */
-    op = copy_extu_tl_i64(&begin_op, op);
-
-    if (type == PLUGIN_GEN_CB_MEM) {
-        /* call */
-        op = copy_call(&begin_op, op, HELPER(plugin_vcpu_mem_cb),
-                       cb->f.vcpu_udata, cb->tcg_flags, cb_idx);
-    }
-
-    return op;
-}
-
-typedef TCGOp *(*inject_fn)(const struct qemu_plugin_dyn_cb *cb,
-                            TCGOp *begin_op, TCGOp *op, int *intp);
-typedef bool (*op_ok_fn)(const TCGOp *op, const struct qemu_plugin_dyn_cb *cb);
-
-static bool op_ok(const TCGOp *op, const struct qemu_plugin_dyn_cb *cb)
-{
-    return true;
-}
-
-static bool op_rw(const TCGOp *op, const struct qemu_plugin_dyn_cb *cb)
-{
-    int w;
-
-    w = op->args[2];
-    return !!(cb->rw & (w + 1));
-}
-
-static inline
-void inject_cb_type(const GArray *cbs, TCGOp *begin_op, inject_fn inject,
-                    op_ok_fn ok)
-{
-    TCGOp *end_op;
-    TCGOp *op;
-    int cb_idx = -1;
-    int i;
-
-    if (!cbs || cbs->len == 0) {
-        rm_ops(begin_op);
-        return;
-    }
-
-    end_op = find_op(begin_op, INDEX_op_plugin_cb_end);
-    tcg_debug_assert(end_op);
-
-    op = end_op;
-    for (i = 0; i < cbs->len; i++) {
-        struct qemu_plugin_dyn_cb *cb =
-            &g_array_index(cbs, struct qemu_plugin_dyn_cb, i);
-
-        if (!ok(begin_op, cb)) {
-            continue;
-        }
-        op = inject(cb, begin_op, op, &cb_idx);
-    }
-    rm_ops_range(begin_op, end_op);
-}
-
-static void
-inject_udata_cb(const GArray *cbs, TCGOp *begin_op)
-{
-    inject_cb_type(cbs, begin_op, append_udata_cb, op_ok);
-}
-
-static void
-inject_inline_cb(const GArray *cbs, TCGOp *begin_op, op_ok_fn ok)
-{
-    inject_cb_type(cbs, begin_op, append_inline_cb, ok);
-}
-
-static void
-inject_mem_cb(const GArray *cbs, TCGOp *begin_op)
-{
-    inject_cb_type(cbs, begin_op, append_mem_cb, op_rw);
-}
-
-/* we could change the ops in place, but we can reuse more code by copying */
-static void inject_mem_helper(TCGOp *begin_op, GArray *arr)
-{
-    TCGOp *orig_op = begin_op;
-    TCGOp *end_op;
-    TCGOp *op;
-
-    end_op = find_op(begin_op, INDEX_op_plugin_cb_end);
-    tcg_debug_assert(end_op);
-
-    /* const ptr */
-    op = copy_const_ptr(&begin_op, end_op, arr);
-
-    /* st_ptr */
-    op = copy_st_ptr(&begin_op, op);
-
-    rm_ops_range(orig_op, end_op);
-}
-
-/*
- * Tracking memory accesses performed from helpers requires extra work.
- * If an instruction is emulated with helpers, we do two things:
- * (1) copy the CB descriptors, and keep track of it so that they can be
- * freed later on, and (2) point CPUState.plugin_mem_cbs to the descriptors, so
- * that we can read them at run-time (i.e. when the helper executes).
- * This run-time access is performed from qemu_plugin_vcpu_mem_cb.
- *
- * Note that plugin_gen_disable_mem_helpers undoes (2). Since it
- * is possible that the code we generate after the instruction is
- * dead, we also add checks before generating tb_exit etc.
- */
-static void inject_mem_enable_helper(struct qemu_plugin_insn *plugin_insn,
-                                     TCGOp *begin_op)
-{
-    GArray *cbs[2];
-    GArray *arr;
-    size_t n_cbs, i;
-
-    cbs[0] = plugin_insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_REGULAR];
-    cbs[1] = plugin_insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_INLINE];
-
-    n_cbs = 0;
-    for (i = 0; i < ARRAY_SIZE(cbs); i++) {
-        n_cbs += cbs[i]->len;
-    }
-
-    plugin_insn->mem_helper = plugin_insn->calls_helpers && n_cbs;
-    if (likely(!plugin_insn->mem_helper)) {
-        rm_ops(begin_op);
-        return;
-    }
-
-    arr = g_array_sized_new(false, false,
-                            sizeof(struct qemu_plugin_dyn_cb), n_cbs);
-
-    for (i = 0; i < ARRAY_SIZE(cbs); i++) {
-        g_array_append_vals(arr, cbs[i]->data, cbs[i]->len);
-    }
-
-    qemu_plugin_add_dyn_cb_arr(arr);
-    inject_mem_helper(begin_op, arr);
-}
-
-static void inject_mem_disable_helper(struct qemu_plugin_insn *plugin_insn,
-                                      TCGOp *begin_op)
-{
-    if (likely(!plugin_insn->mem_helper)) {
-        rm_ops(begin_op);
-        return;
-    }
-    inject_mem_helper(begin_op, NULL);
-}
-
-/* called before finishing a TB with exit_tb, goto_tb or goto_ptr */
-void plugin_gen_disable_mem_helpers(void)
-{
-    TCGv_ptr ptr;
-
-    if (likely(tcg_ctx->plugin_insn == NULL ||
-               !tcg_ctx->plugin_insn->mem_helper)) {
-        return;
-    }
-    ptr = tcg_const_ptr(NULL);
-    tcg_gen_st_ptr(ptr, cpu_env, offsetof(CPUState, plugin_mem_cbs) -
-                                 offsetof(ArchCPU, env));
-    tcg_temp_free_ptr(ptr);
-    tcg_ctx->plugin_insn->mem_helper = false;
-}
-
-static void plugin_gen_tb_udata(const struct qemu_plugin_tb *ptb,
-                                TCGOp *begin_op)
-{
-    inject_udata_cb(ptb->cbs[PLUGIN_CB_REGULAR], begin_op);
-}
-
-static void plugin_gen_tb_inline(const struct qemu_plugin_tb *ptb,
-                                 TCGOp *begin_op)
-{
-    inject_inline_cb(ptb->cbs[PLUGIN_CB_INLINE], begin_op, op_ok);
-}
-
-static void plugin_gen_insn_udata(const struct qemu_plugin_tb *ptb,
-                                  TCGOp *begin_op, int insn_idx)
-{
-    struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx);
-
-    inject_udata_cb(insn->cbs[PLUGIN_CB_INSN][PLUGIN_CB_REGULAR], begin_op);
-}
-
-static void plugin_gen_insn_inline(const struct qemu_plugin_tb *ptb,
-                                   TCGOp *begin_op, int insn_idx)
-{
-    struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx);
-    inject_inline_cb(insn->cbs[PLUGIN_CB_INSN][PLUGIN_CB_INLINE],
-                     begin_op, op_ok);
-}
-
-static void plugin_gen_mem_regular(const struct qemu_plugin_tb *ptb,
-                                   TCGOp *begin_op, int insn_idx)
-{
-    struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx);
-    inject_mem_cb(insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_REGULAR], begin_op);
-}
-
-static void plugin_gen_mem_inline(const struct qemu_plugin_tb *ptb,
-                                  TCGOp *begin_op, int insn_idx)
-{
-    const GArray *cbs;
-    struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx);
-
-    cbs = insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_INLINE];
-    inject_inline_cb(cbs, begin_op, op_rw);
-}
-
-static void plugin_gen_enable_mem_helper(const struct qemu_plugin_tb *ptb,
-                                         TCGOp *begin_op, int insn_idx)
-{
-    struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx);
-    inject_mem_enable_helper(insn, begin_op);
-}
-
-static void plugin_gen_disable_mem_helper(const struct qemu_plugin_tb *ptb,
-                                          TCGOp *begin_op, int insn_idx)
-{
-    struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx);
-    inject_mem_disable_helper(insn, begin_op);
-}
-
-static void plugin_inject_cb(const struct qemu_plugin_tb *ptb, TCGOp *begin_op,
-                             int insn_idx)
-{
-    enum plugin_gen_from from = begin_op->args[0];
-    enum plugin_gen_cb type = begin_op->args[1];
-
-    switch (from) {
-    case PLUGIN_GEN_FROM_TB:
-        switch (type) {
-        case PLUGIN_GEN_CB_UDATA:
-            plugin_gen_tb_udata(ptb, begin_op);
-            return;
-        case PLUGIN_GEN_CB_INLINE:
-            plugin_gen_tb_inline(ptb, begin_op);
-            return;
-        default:
-            g_assert_not_reached();
-        }
-    case PLUGIN_GEN_FROM_INSN:
-        switch (type) {
-        case PLUGIN_GEN_CB_UDATA:
-            plugin_gen_insn_udata(ptb, begin_op, insn_idx);
-            return;
-        case PLUGIN_GEN_CB_INLINE:
-            plugin_gen_insn_inline(ptb, begin_op, insn_idx);
-            return;
-        case PLUGIN_GEN_ENABLE_MEM_HELPER:
-            plugin_gen_enable_mem_helper(ptb, begin_op, insn_idx);
-            return;
-        default:
-            g_assert_not_reached();
-        }
-    case PLUGIN_GEN_FROM_MEM:
-        switch (type) {
-        case PLUGIN_GEN_CB_MEM:
-            plugin_gen_mem_regular(ptb, begin_op, insn_idx);
-            return;
-        case PLUGIN_GEN_CB_INLINE:
-            plugin_gen_mem_inline(ptb, begin_op, insn_idx);
-            return;
-        default:
-            g_assert_not_reached();
-        }
-    case PLUGIN_GEN_AFTER_INSN:
-        switch (type) {
-        case PLUGIN_GEN_DISABLE_MEM_HELPER:
-            plugin_gen_disable_mem_helper(ptb, begin_op, insn_idx);
-            return;
-        default:
-            g_assert_not_reached();
-        }
-    default:
-        g_assert_not_reached();
-    }
-}
-
-/* #define DEBUG_PLUGIN_GEN_OPS */
-static void pr_ops(void)
-{
-#ifdef DEBUG_PLUGIN_GEN_OPS
-    TCGOp *op;
-    int i = 0;
-
-    QTAILQ_FOREACH(op, &tcg_ctx->ops, link) {
-        const char *name = "";
-        const char *type = "";
-
-        if (op->opc == INDEX_op_plugin_cb_start) {
-            switch (op->args[0]) {
-            case PLUGIN_GEN_FROM_TB:
-                name = "tb";
-                break;
-            case PLUGIN_GEN_FROM_INSN:
-                name = "insn";
-                break;
-            case PLUGIN_GEN_FROM_MEM:
-                name = "mem";
-                break;
-            case PLUGIN_GEN_AFTER_INSN:
-                name = "after insn";
-                break;
-            default:
-                break;
-            }
-            switch (op->args[1]) {
-            case PLUGIN_GEN_CB_UDATA:
-                type = "udata";
-                break;
-            case PLUGIN_GEN_CB_INLINE:
-                type = "inline";
-                break;
-            case PLUGIN_GEN_CB_MEM:
-                type = "mem";
-                break;
-            case PLUGIN_GEN_ENABLE_MEM_HELPER:
-                type = "enable mem helper";
-                break;
-            case PLUGIN_GEN_DISABLE_MEM_HELPER:
-                type = "disable mem helper";
-                break;
-            default:
-                break;
-            }
-        }
-        printf("op[%2i]: %s %s %s\n", i, tcg_op_defs[op->opc].name, name, type);
-        i++;
-    }
-#endif
-}
-
-static void plugin_gen_inject(const struct qemu_plugin_tb *plugin_tb)
-{
-    TCGOp *op;
-    int insn_idx;
-
-    pr_ops();
-    insn_idx = -1;
-    QSIMPLEQ_FOREACH(op, &tcg_ctx->plugin_ops, plugin_link) {
-        enum plugin_gen_from from = op->args[0];
-        enum plugin_gen_cb type = op->args[1];
-
-        tcg_debug_assert(op->opc == INDEX_op_plugin_cb_start);
-        /* ENABLE_MEM_HELPER is the first callback of an instruction */
-        if (from == PLUGIN_GEN_FROM_INSN &&
-            type == PLUGIN_GEN_ENABLE_MEM_HELPER) {
-            insn_idx++;
-        }
-        plugin_inject_cb(plugin_tb, op, insn_idx);
-    }
-    pr_ops();
-}
-
-bool plugin_gen_tb_start(CPUState *cpu, const TranslationBlock *tb)
-{
-    struct qemu_plugin_tb *ptb = tcg_ctx->plugin_tb;
-    bool ret = false;
-
-    if (test_bit(QEMU_PLUGIN_EV_VCPU_TB_TRANS, cpu->plugin_mask)) {
-        ret = true;
-
-        QSIMPLEQ_INIT(&tcg_ctx->plugin_ops);
-        ptb->vaddr = tb->pc;
-        ptb->vaddr2 = -1;
-        get_page_addr_code_hostp(cpu->env_ptr, tb->pc, &ptb->haddr1);
-        ptb->haddr2 = NULL;
-
-        plugin_gen_empty_callback(PLUGIN_GEN_FROM_TB);
-    }
-    return ret;
-}
-
-void plugin_gen_insn_start(CPUState *cpu, const DisasContextBase *db)
-{
-    struct qemu_plugin_tb *ptb = tcg_ctx->plugin_tb;
-    struct qemu_plugin_insn *pinsn;
-
-    pinsn = qemu_plugin_tb_insn_get(ptb);
-    tcg_ctx->plugin_insn = pinsn;
-    pinsn->vaddr = db->pc_next;
-    plugin_gen_empty_callback(PLUGIN_GEN_FROM_INSN);
-
-    /*
-     * Detect page crossing to get the new host address.
-     * Note that we skip this when haddr1 == NULL, e.g. when we're
-     * fetching instructions from a region not backed by RAM.
-     */
-    if (likely(ptb->haddr1 != NULL && ptb->vaddr2 == -1) &&
-        unlikely((db->pc_next & TARGET_PAGE_MASK) !=
-                 (db->pc_first & TARGET_PAGE_MASK))) {
-        get_page_addr_code_hostp(cpu->env_ptr, db->pc_next,
-                                 &ptb->haddr2);
-        ptb->vaddr2 = db->pc_next;
-    }
-    if (likely(ptb->vaddr2 == -1)) {
-        pinsn->haddr = ptb->haddr1 + pinsn->vaddr - ptb->vaddr;
-    } else {
-        pinsn->haddr = ptb->haddr2 + pinsn->vaddr - ptb->vaddr2;
-    }
-}
-
-void plugin_gen_insn_end(void)
-{
-    plugin_gen_empty_callback(PLUGIN_GEN_AFTER_INSN);
-}
-
-void plugin_gen_tb_end(CPUState *cpu)
-{
-    struct qemu_plugin_tb *ptb = tcg_ctx->plugin_tb;
-    int i;
-
-    /* collect instrumentation requests */
-    qemu_plugin_tb_trans_cb(cpu, ptb);
-
-    /* inject the instrumentation at the appropriate places */
-    plugin_gen_inject(ptb);
-
-    /* clean up */
-    for (i = 0; i < PLUGIN_N_CB_SUBTYPES; i++) {
-        if (ptb->cbs[i]) {
-            g_array_set_size(ptb->cbs[i], 0);
-        }
-    }
-    ptb->n = 0;
-    tcg_ctx->plugin_insn = NULL;
-}
@@ -1,5 +0,0 @@
-#ifdef CONFIG_PLUGIN
-/* Note: no TCG flags because those are overwritten later */
-DEF_HELPER_2(plugin_vcpu_udata_cb, void, i32, ptr)
-DEF_HELPER_4(plugin_vcpu_mem_cb, void, i32, i32, i64, ptr)
-#endif
@@ -25,6 +25,7 @@

 #include "qemu/osdep.h"
 #include "sysemu/accel.h"
+#include "sysemu/sysemu.h"
 #include "sysemu/tcg.h"
 #include "qom/object.h"
 #include "cpu.h"
@@ -1156,6 +1156,23 @@ void tcg_exec_init(unsigned long tb_size)
 #endif
 }

+/*
+ * Allocate a new translation block. Flush the translation buffer if
+ * too many translation blocks or too much generated code.
+ */
+static TranslationBlock *tb_alloc(target_ulong pc)
+{
+    TranslationBlock *tb;
+
+    assert_memory_lock();
+
+    tb = tcg_tb_alloc(tcg_ctx);
+    if (unlikely(tb == NULL)) {
+        return NULL;
+    }
+    return tb;
+}
+
 /* call with @p->lock held */
 static inline void invalidate_page_bitmap(PageDesc *p)
 {
@@ -1214,8 +1231,6 @@ static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data)
 /* flush all the translation blocks */
 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
 {
-    bool did_flush = false;
-
    mmap_lock();
    /* If it is already been done on request of another CPU,
     * just retry.
@@ -1223,7 +1238,6 @@ static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
    if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
        goto done;
    }
-    did_flush = true;

    if (DEBUG_TB_FLUSH_GATE) {
        size_t nb_tbs = tcg_nb_tbs();
@@ -1248,22 +1262,14 @@ static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)

 done:
    mmap_unlock();
-    if (did_flush) {
-        qemu_plugin_flush_cb();
-    }
 }

 void tb_flush(CPUState *cpu)
 {
    if (tcg_enabled()) {
        unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count);
-
-        if (cpu_in_exclusive_context(cpu)) {
-            do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
-        } else {
-            async_safe_run_on_cpu(cpu, do_tb_flush,
-                                  RUN_ON_CPU_HOST_INT(tb_flush_count));
-        }
+        async_safe_run_on_cpu(cpu, do_tb_flush,
+                              RUN_ON_CPU_HOST_INT(tb_flush_count));
    }
 }

@@ -1675,7 +1681,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
    TCGProfile *prof = &tcg_ctx->prof;
    int64_t ti;
 #endif
-
    assert_memory_lock();

    phys_pc = get_page_addr_code(env, pc);
@@ -1701,7 +1706,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
    }

 buffer_overflow:
-    tb = tcg_tb_alloc(tcg_ctx);
+    tb = tb_alloc(pc);
    if (unlikely(!tb)) {
        /* flush must be done */
        tb_flush(cpu);
@@ -1717,7 +1722,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
    tb->cs_base = cs_base;
    tb->flags = flags;
    tb->cflags = cflags;
-    tb->orig_tb = NULL;
    tb->trace_vcpu_dstate = *cpu->trace_dstate;
    tcg_ctx->tb_cflags = cflags;
 tb_overflow:
@@ -1885,7 +1889,7 @@ static void
 tb_invalidate_phys_page_range__locked(struct page_collection *pages,
                                      PageDesc *p, tb_page_addr_t start,
                                      tb_page_addr_t end,
-                                      uintptr_t retaddr)
+                                      int is_cpu_write_access)
 {
    TranslationBlock *tb;
    tb_page_addr_t tb_start, tb_end;
@@ -1893,9 +1897,9 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
 #ifdef TARGET_HAS_PRECISE_SMC
    CPUState *cpu = current_cpu;
    CPUArchState *env = NULL;
-    bool current_tb_not_found = retaddr != 0;
-    bool current_tb_modified = false;
+    int current_tb_not_found = is_cpu_write_access;
    TranslationBlock *current_tb = NULL;
+    int current_tb_modified = 0;
    target_ulong current_pc = 0;
    target_ulong current_cs_base = 0;
    uint32_t current_flags = 0;
@@ -1927,21 +1931,24 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
        if (!(tb_end <= start || tb_start >= end)) {
 #ifdef TARGET_HAS_PRECISE_SMC
            if (current_tb_not_found) {
-                current_tb_not_found = false;
-                /* now we have a real cpu fault */
-                current_tb = tcg_tb_lookup(retaddr);
+                current_tb_not_found = 0;
+                current_tb = NULL;
+                if (cpu->mem_io_pc) {
+                    /* now we have a real cpu fault */
+                    current_tb = tcg_tb_lookup(cpu->mem_io_pc);
+                }
            }
            if (current_tb == tb &&
                (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
-                /*
-                 * If we are modifying the current TB, we must stop
-                 * its execution. We could be more precise by checking
-                 * that the modification is after the current PC, but it
-                 * would require a specialized function to partially
-                 * restore the CPU state.
-                 */
-                current_tb_modified = true;
-                cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
+                /* If we are modifying the current TB, we must stop
+                its execution. We could be more precise by checking
+                that the modification is after the current PC, but it
+                would require a specialized function to partially
+                restore the CPU state */
+
+                current_tb_modified = 1;
+                cpu_restore_state_from_tb(cpu, current_tb,
+                                          cpu->mem_io_pc, true);
                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                                     &current_flags);
            }
@@ -1976,7 +1983,8 @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages,
 *
 * Called with mmap_lock held for user-mode emulation
 */
-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
+                                   int is_cpu_write_access)
 {
    struct page_collection *pages;
    PageDesc *p;
@@ -1988,7 +1996,8 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
        return;
    }
    pages = page_collection_lock(start, end);
-    tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
+    tb_invalidate_phys_page_range__locked(pages, p, start, end,
+                                          is_cpu_write_access);
    page_collection_unlock(pages);
 }

@@ -2035,8 +2044,7 @@ void tb_invalidate_phys_range(target_ulong start, target_ulong end)
 * Call with all @pages in the range [@start, @start + len[ locked.
 */
 void tb_invalidate_phys_page_fast(struct page_collection *pages,
-                                  tb_page_addr_t start, int len,
-                                  uintptr_t retaddr)
+                                  tb_page_addr_t start, int len)
 {
    PageDesc *p;

@@ -2063,8 +2071,7 @@ void tb_invalidate_phys_page_fast(struct page_collection *pages,
        }
    } else {
    do_invalidate:
-        tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
-                                              retaddr);
+        tb_invalidate_phys_page_range__locked(pages, p, start, start + len, 1);
    }
 }
 #else
@@ -2138,16 +2145,16 @@ static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
 #endif

 /* user-mode: call with mmap_lock held */
-void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
+void tb_check_watchpoint(CPUState *cpu)
 {
    TranslationBlock *tb;

    assert_memory_lock();

-    tb = tcg_tb_lookup(retaddr);
+    tb = tcg_tb_lookup(cpu->mem_io_pc);
    if (tb) {
        /* We can use retranslation to find the PC.  */
-        cpu_restore_state_from_tb(cpu, tb, retaddr, true);
+        cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc, true);
        tb_phys_invalidate(tb, -1);
    } else {
        /* The exception probably happened in a helper.  The CPU state should
@@ -27,10 +27,10 @@ struct page_collection *page_collection_lock(tb_page_addr_t start,
                                             tb_page_addr_t end);
 void page_collection_unlock(struct page_collection *set);
 void tb_invalidate_phys_page_fast(struct page_collection *pages,
-                                  tb_page_addr_t start, int len,
-                                  uintptr_t retaddr);
-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end);
-void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr);
+                                  tb_page_addr_t start, int len);
+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
+                                   int is_cpu_write_access);
+void tb_check_watchpoint(CPUState *cpu);

 #ifdef CONFIG_USER_ONLY
 int page_unprotect(target_ulong address, uintptr_t pc);
@@ -16,7 +16,6 @@
 #include "exec/gen-icount.h"
 #include "exec/log.h"
 #include "exec/translator.h"
-#include "exec/plugin-gen.h"

 /* Pairs with tcg_clear_temp_count.
   To be called by #TranslatorOps.{translate_insn,tb_stop} if
@@ -35,7 +34,6 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
                     CPUState *cpu, TranslationBlock *tb, int max_insns)
 {
    int bp_insn = 0;
-    bool plugin_enabled;

    /* Initialize DisasContext */
    db->tb = tb;
@@ -57,17 +55,11 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
    ops->tb_start(db, cpu);
    tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */

-    plugin_enabled = plugin_gen_tb_start(cpu, tb);
-
    while (true) {
        db->num_insns++;
        ops->insn_start(db, cpu);
        tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */

-        if (plugin_enabled) {
-            plugin_gen_insn_start(cpu, db);
-        }
-
        /* Pass breakpoint hits to target for further processing */
        if (!db->singlestep_enabled
            && unlikely(!QTAILQ_EMPTY(&cpu->breakpoints))) {
@@ -98,6 +90,7 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
            /* Accept I/O on the last instruction.  */
            gen_io_start();
            ops->translate_insn(db, cpu);
+            gen_io_end();
        } else {
            ops->translate_insn(db, cpu);
        }
@@ -107,14 +100,6 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
            break;
        }

-        /*
-         * We can't instrument after instructions that change control
-         * flow although this only really affects post-load operations.
-         */
-        if (plugin_enabled) {
-            plugin_gen_insn_end();
-        }
-
        /* Stop translation if the output buffer is full,
           or we have executed all of the allowed instructions.  */
        if (tcg_op_buf_full() || db->num_insns >= db->max_insns) {
@@ -127,10 +112,6 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
    ops->tb_stop(db, cpu);
    gen_tb_end(db->tb, db->num_insns - bp_insn);

-    if (plugin_enabled) {
-        plugin_gen_tb_end(cpu);
-    }
-
    /* The disas_log hook may use these values rather than recompute.  */
    db->tb->size = db->pc_next - db->pc_first;
    db->tb->icount = db->num_insns;
@@ -1,5 +1,5 @@
 #include "qemu/osdep.h"
-#include "hw/core/cpu.h"
+#include "qom/cpu.h"
 #include "sysemu/replay.h"
 #include "sysemu/sysemu.h"

@@ -188,38 +188,6 @@ static inline int handle_cpu_signal(uintptr_t pc, siginfo_t *info,
    g_assert_not_reached();
 }

-void *probe_access(CPUArchState *env, target_ulong addr, int size,
-                   MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
-{
-    int flags;
-
-    g_assert(-(addr | TARGET_PAGE_MASK) >= size);
-
-    switch (access_type) {
-    case MMU_DATA_STORE:
-        flags = PAGE_WRITE;
-        break;
-    case MMU_DATA_LOAD:
-        flags = PAGE_READ;
-        break;
-    case MMU_INST_FETCH:
-        flags = PAGE_EXEC;
-        break;
-    default:
-        g_assert_not_reached();
-    }
-
-    if (!guest_addr_valid(addr) || page_check_range(addr, size, flags) < 0) {
-        CPUState *cpu = env_cpu(env);
-        CPUClass *cc = CPU_GET_CLASS(cpu);
-        cc->tlb_fill(cpu, addr, size, access_type, MMU_USER_IDX, false,
-                     retaddr);
-        g_assert_not_reached();
-    }
-
-    return size ? g2h(addr) : NULL;
-}
-
 #if defined(__i386__)

 #if defined(__NetBSD__)
@@ -751,13 +719,10 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 #define ATOMIC_MMU_DECLS do {} while (0)
 #define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC())
 #define ATOMIC_MMU_CLEANUP do { clear_helper_retaddr(); } while (0)
-#define ATOMIC_MMU_IDX MMU_USER_IDX

 #define ATOMIC_NAME(X)   HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
 #define EXTRA_ARGS

-#include "atomic_common.inc.c"
-
 #define DATA_SIZE 1
 #include "atomic_template.h"

@@ -38,10 +38,6 @@
 int graphic_width = 1024;
 int graphic_height = 768;
 int graphic_depth = 8;
-#elif defined(TARGET_M68K)
-int graphic_width = 800;
-int graphic_height = 600;
-int graphic_depth = 8;
 #else
 int graphic_width = 800;
 int graphic_height = 600;
@@ -110,3 +106,14 @@ int xen_available(void)
    return 0;
 #endif
 }
+
+
+TargetInfo *qmp_query_target(Error **errp)
+{
+    TargetInfo *info = g_malloc0(sizeof(*info));
+
+    info->arch = qapi_enum_parse(&SysEmuTarget_lookup, TARGET_NAME, -1,
+                                 &error_abort);
+
+    return info;
+}
@@ -2,6 +2,7 @@ common-obj-y = audio.o audio_legacy.o noaudio.o wavaudio.o mixeng.o
 common-obj-$(CONFIG_SPICE) += spiceaudio.o
 common-obj-$(CONFIG_AUDIO_COREAUDIO) += coreaudio.o
 common-obj-$(CONFIG_AUDIO_DSOUND) += dsoundaudio.o
+common-obj-$(CONFIG_AUDIO_PT_INT) += audio_pt_int.o
 common-obj-$(CONFIG_AUDIO_WIN_INT) += audio_win_int.o
 common-obj-y += wavcapture.o

@@ -39,11 +39,13 @@ struct pollhlp {
    struct pollfd *pfds;
    int count;
    int mask;
-    AudioState *s;
 };

 typedef struct ALSAVoiceOut {
    HWVoiceOut hw;
+    int wpos;
+    int pending;
+    void *pcm_buf;
    snd_pcm_t *handle;
    struct pollhlp pollhlp;
    Audiodev *dev;
@@ -52,6 +54,7 @@ typedef struct ALSAVoiceOut {
 typedef struct ALSAVoiceIn {
    HWVoiceIn hw;
    snd_pcm_t *handle;
+    void *pcm_buf;
    struct pollhlp pollhlp;
    Audiodev *dev;
 } ALSAVoiceIn;
@@ -196,11 +199,11 @@ static void alsa_poll_handler (void *opaque)
        break;

    case SND_PCM_STATE_PREPARED:
-        audio_run(hlp->s, "alsa run (prepared)");
+        audio_run ("alsa run (prepared)");
        break;

    case SND_PCM_STATE_RUNNING:
-        audio_run(hlp->s, "alsa run (running)");
+        audio_run ("alsa run (running)");
        break;

    default:
@@ -266,6 +269,11 @@ static int alsa_poll_in (HWVoiceIn *hw)
    return alsa_poll_helper (alsa->handle, &alsa->pollhlp, POLLIN);
 }

+static int alsa_write (SWVoiceOut *sw, void *buf, int len)
+{
+    return audio_pcm_sw_write (sw, buf, len);
+}
+
 static snd_pcm_format_t aud_to_alsafmt (AudioFormat fmt, int endianness)
 {
    switch (fmt) {
@@ -493,6 +501,13 @@ static int alsa_open(bool in, struct alsa_params_req *req,
        goto err;
    }

+    if (nchannels != 1 && nchannels != 2) {
+        alsa_logerr2 (err, typ,
+                      "Can not handle obtained number of channels %d\n",
+                      nchannels);
+        goto err;
+    }
+
    if (apdo->buffer_length) {
        int dir = 0;
        unsigned int btime = apdo->buffer_length;
@@ -591,64 +606,102 @@ static int alsa_open(bool in, struct alsa_params_req *req,
    return -1;
 }

-static size_t alsa_write(HWVoiceOut *hw, void *buf, size_t len)
+static snd_pcm_sframes_t alsa_get_avail (snd_pcm_t *handle)
 {
-    ALSAVoiceOut *alsa = (ALSAVoiceOut *) hw;
-    size_t pos = 0;
-    size_t len_frames = len / hw->info.bytes_per_frame;
+    snd_pcm_sframes_t avail;

-    while (len_frames) {
-        char *src = advance(buf, pos);
-        snd_pcm_sframes_t written;
-
-        written = snd_pcm_writei(alsa->handle, src, len_frames);
-
-        if (written <= 0) {
-            switch (written) {
-            case 0:
-                trace_alsa_wrote_zero(len_frames);
-                return pos;
-
-            case -EPIPE:
-                if (alsa_recover(alsa->handle)) {
-                    alsa_logerr(written, "Failed to write %zu frames\n",
-                                len_frames);
-                    return pos;
-                }
-                trace_alsa_xrun_out();
-                continue;
-
-            case -ESTRPIPE:
-                /*
-                 * stream is suspended and waiting for an application
-                 * recovery
-                 */
-                if (alsa_resume(alsa->handle)) {
-                    alsa_logerr(written, "Failed to write %zu frames\n",
-                                len_frames);
-                    return pos;
-                }
-                trace_alsa_resume_out();
-                continue;
-
-            case -EAGAIN:
-                return pos;
-
-            default:
-                alsa_logerr(written, "Failed to write %zu frames from %p\n",
-                            len, src);
-                return pos;
+    avail = snd_pcm_avail_update (handle);
+    if (avail < 0) {
+        if (avail == -EPIPE) {
+            if (!alsa_recover (handle)) {
+                avail = snd_pcm_avail_update (handle);
            }
        }

-        pos += written * hw->info.bytes_per_frame;
-        if (written < len_frames) {
-            break;
+        if (avail < 0) {
+            alsa_logerr (avail,
+                         "Could not obtain number of available frames\n");
+            return -1;
        }
-        len_frames -= written;
    }

-    return pos;
+    return avail;
+}
+
+static void alsa_write_pending (ALSAVoiceOut *alsa)
+{
+    HWVoiceOut *hw = &alsa->hw;
+
+    while (alsa->pending) {
+        int left_till_end_samples = hw->samples - alsa->wpos;
+        int len = audio_MIN (alsa->pending, left_till_end_samples);
+        char *src = advance (alsa->pcm_buf, alsa->wpos << hw->info.shift);
+
+        while (len) {
+            snd_pcm_sframes_t written;
+
+            written = snd_pcm_writei (alsa->handle, src, len);
+
+            if (written <= 0) {
+                switch (written) {
+                case 0:
+                    trace_alsa_wrote_zero(len);
+                    return;
+
+                case -EPIPE:
+                    if (alsa_recover (alsa->handle)) {
+                        alsa_logerr (written, "Failed to write %d frames\n",
+                                     len);
+                        return;
+                    }
+                    trace_alsa_xrun_out();
+                    continue;
+
+                case -ESTRPIPE:
+                    /* stream is suspended and waiting for an
+                       application recovery */
+                    if (alsa_resume (alsa->handle)) {
+                        alsa_logerr (written, "Failed to write %d frames\n",
+                                     len);
+                        return;
+                    }
+                    trace_alsa_resume_out();
+                    continue;
+
+                case -EAGAIN:
+                    return;
+
+                default:
+                    alsa_logerr (written, "Failed to write %d frames from %p\n",
+                                 len, src);
+                    return;
+                }
+            }
+
+            alsa->wpos = (alsa->wpos + written) % hw->samples;
+            alsa->pending -= written;
+            len -= written;
+        }
+    }
+}
+
+static int alsa_run_out (HWVoiceOut *hw, int live)
+{
+    ALSAVoiceOut *alsa = (ALSAVoiceOut *) hw;
+    int decr;
+    snd_pcm_sframes_t avail;
+
+    avail = alsa_get_avail (alsa->handle);
+    if (avail < 0) {
+        dolog ("Could not get number of available playback frames\n");
+        return 0;
+    }
+
+    decr = audio_MIN (live, avail);
+    decr = audio_pcm_hw_clip_out (hw, alsa->pcm_buf, decr, alsa->pending);
+    alsa->pending += decr;
+    alsa_write_pending (alsa);
+    return decr;
 }

 static void alsa_fini_out (HWVoiceOut *hw)
@@ -657,6 +710,9 @@ static void alsa_fini_out (HWVoiceOut *hw)

    ldebug ("alsa_fini\n");
    alsa_anal_close (&alsa->handle, &alsa->pollhlp);
+
+    g_free(alsa->pcm_buf);
+    alsa->pcm_buf = NULL;
 }

 static int alsa_init_out(HWVoiceOut *hw, struct audsettings *as,
@@ -685,7 +741,14 @@ static int alsa_init_out(HWVoiceOut *hw, struct audsettings *as,
    audio_pcm_init_info (&hw->info, &obt_as);
    hw->samples = obt.samples;

-    alsa->pollhlp.s = hw->s;
+    alsa->pcm_buf = audio_calloc(__func__, obt.samples, 1 << hw->info.shift);
+    if (!alsa->pcm_buf) {
+        dolog ("Could not allocate DAC buffer (%d samples, each %d bytes)\n",
+               hw->samples, 1 << hw->info.shift);
+        alsa_anal_close1 (&handle);
+        return -1;
+    }
+
    alsa->handle = handle;
    alsa->dev = dev;
    return 0;
@@ -724,28 +787,34 @@ static int alsa_voice_ctl (snd_pcm_t *handle, const char *typ, int ctl)
    return 0;
 }

-static void alsa_enable_out(HWVoiceOut *hw, bool enable)
+static int alsa_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
    ALSAVoiceOut *alsa = (ALSAVoiceOut *) hw;
    AudiodevAlsaPerDirectionOptions *apdo = alsa->dev->u.alsa.out;

-    if (enable) {
-        bool poll_mode = apdo->try_poll;
+    switch (cmd) {
+    case VOICE_ENABLE:
+        {
+            bool poll_mode = apdo->try_poll;

-        ldebug("enabling voice\n");
-        if (poll_mode && alsa_poll_out(hw)) {
-            poll_mode = 0;
+            ldebug ("enabling voice\n");
+            if (poll_mode && alsa_poll_out (hw)) {
+                poll_mode = 0;
+            }
+            hw->poll_mode = poll_mode;
+            return alsa_voice_ctl (alsa->handle, "playback", VOICE_CTL_PREPARE);
        }
-        hw->poll_mode = poll_mode;
-        alsa_voice_ctl(alsa->handle, "playback", VOICE_CTL_PREPARE);
-    } else {
-        ldebug("disabling voice\n");
+
+    case VOICE_DISABLE:
+        ldebug ("disabling voice\n");
        if (hw->poll_mode) {
            hw->poll_mode = 0;
-            alsa_fini_poll(&alsa->pollhlp);
+            alsa_fini_poll (&alsa->pollhlp);
        }
-        alsa_voice_ctl(alsa->handle, "playback", VOICE_CTL_PAUSE);
+        return alsa_voice_ctl (alsa->handle, "playback", VOICE_CTL_PAUSE);
    }
+
+    return -1;
 }

 static int alsa_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
@@ -773,7 +842,14 @@ static int alsa_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
    audio_pcm_init_info (&hw->info, &obt_as);
    hw->samples = obt.samples;

-    alsa->pollhlp.s = hw->s;
+    alsa->pcm_buf = audio_calloc(__func__, hw->samples, 1 << hw->info.shift);
+    if (!alsa->pcm_buf) {
+        dolog ("Could not allocate ADC buffer (%d samples, each %d bytes)\n",
+               hw->samples, 1 << hw->info.shift);
+        alsa_anal_close1 (&handle);
+        return -1;
+    }
+
    alsa->handle = handle;
    alsa->dev = dev;
    return 0;
@@ -784,74 +860,165 @@ static void alsa_fini_in (HWVoiceIn *hw)
    ALSAVoiceIn *alsa = (ALSAVoiceIn *) hw;

    alsa_anal_close (&alsa->handle, &alsa->pollhlp);
+
+    g_free(alsa->pcm_buf);
+    alsa->pcm_buf = NULL;
 }

-static size_t alsa_read(HWVoiceIn *hw, void *buf, size_t len)
+static int alsa_run_in (HWVoiceIn *hw)
 {
    ALSAVoiceIn *alsa = (ALSAVoiceIn *) hw;
-    size_t pos = 0;
+    int hwshift = hw->info.shift;
+    int i;
+    int live = audio_pcm_hw_get_live_in (hw);
+    int dead = hw->samples - live;
+    int decr;
+    struct {
+        int add;
+        int len;
+    } bufs[2] = {
+        { .add = hw->wpos, .len = 0 },
+        { .add = 0,        .len = 0 }
+    };
+    snd_pcm_sframes_t avail;
+    snd_pcm_uframes_t read_samples = 0;

-    while (len) {
-        void *dst = advance(buf, pos);
-        snd_pcm_sframes_t nread;
-
-        nread = snd_pcm_readi(
-            alsa->handle, dst, len / hw->info.bytes_per_frame);
-
-        if (nread <= 0) {
-            switch (nread) {
-            case 0:
-                trace_alsa_read_zero(len);
-                return pos;;
-
-            case -EPIPE:
-                if (alsa_recover(alsa->handle)) {
-                    alsa_logerr(nread, "Failed to read %zu frames\n", len);
-                    return pos;
-                }
-                trace_alsa_xrun_in();
-                continue;
-
-            case -EAGAIN:
-                return pos;
-
-            default:
-                alsa_logerr(nread, "Failed to read %zu frames to %p\n",
-                            len, dst);
-                return pos;;
-            }
-        }
-
-        pos += nread * hw->info.bytes_per_frame;
-        len -= nread * hw->info.bytes_per_frame;
+    if (!dead) {
+        return 0;
    }

-    return pos;
+    avail = alsa_get_avail (alsa->handle);
+    if (avail < 0) {
+        dolog ("Could not get number of captured frames\n");
+        return 0;
+    }
+
+    if (!avail) {
+        snd_pcm_state_t state;
+
+        state = snd_pcm_state (alsa->handle);
+        switch (state) {
+        case SND_PCM_STATE_PREPARED:
+            avail = hw->samples;
+            break;
+        case SND_PCM_STATE_SUSPENDED:
+            /* stream is suspended and waiting for an application recovery */
+            if (alsa_resume (alsa->handle)) {
+                dolog ("Failed to resume suspended input stream\n");
+                return 0;
+            }
+            trace_alsa_resume_in();
+            break;
+        default:
+            trace_alsa_no_frames(state);
+            return 0;
+        }
+    }
+
+    decr = audio_MIN (dead, avail);
+    if (!decr) {
+        return 0;
+    }
+
+    if (hw->wpos + decr > hw->samples) {
+        bufs[0].len = (hw->samples - hw->wpos);
+        bufs[1].len = (decr - (hw->samples - hw->wpos));
+    }
+    else {
+        bufs[0].len = decr;
+    }
+
+    for (i = 0; i < 2; ++i) {
+        void *src;
+        struct st_sample *dst;
+        snd_pcm_sframes_t nread;
+        snd_pcm_uframes_t len;
+
+        len = bufs[i].len;
+
+        src = advance (alsa->pcm_buf, bufs[i].add << hwshift);
+        dst = hw->conv_buf + bufs[i].add;
+
+        while (len) {
+            nread = snd_pcm_readi (alsa->handle, src, len);
+
+            if (nread <= 0) {
+                switch (nread) {
+                case 0:
+                    trace_alsa_read_zero(len);
+                    goto exit;
+
+                case -EPIPE:
+                    if (alsa_recover (alsa->handle)) {
+                        alsa_logerr (nread, "Failed to read %ld frames\n", len);
+                        goto exit;
+                    }
+                    trace_alsa_xrun_in();
+                    continue;
+
+                case -EAGAIN:
+                    goto exit;
+
+                default:
+                    alsa_logerr (
+                        nread,
+                        "Failed to read %ld frames from %p\n",
+                        len,
+                        src
+                        );
+                    goto exit;
+                }
+            }
+
+            hw->conv (dst, src, nread);
+
+            src = advance (src, nread << hwshift);
+            dst += nread;
+
+            read_samples += nread;
+            len -= nread;
+        }
+    }
+
+ exit:
+    hw->wpos = (hw->wpos + read_samples) % hw->samples;
+    return read_samples;
 }

-static void alsa_enable_in(HWVoiceIn *hw, bool enable)
+static int alsa_read (SWVoiceIn *sw, void *buf, int size)
+{
+    return audio_pcm_sw_read (sw, buf, size);
+}
+
+static int alsa_ctl_in (HWVoiceIn *hw, int cmd, ...)
 {
    ALSAVoiceIn *alsa = (ALSAVoiceIn *) hw;
    AudiodevAlsaPerDirectionOptions *apdo = alsa->dev->u.alsa.in;

-    if (enable) {
-        bool poll_mode = apdo->try_poll;
+    switch (cmd) {
+    case VOICE_ENABLE:
+        {
+            bool poll_mode = apdo->try_poll;

-        ldebug("enabling voice\n");
-        if (poll_mode && alsa_poll_in(hw)) {
-            poll_mode = 0;
+            ldebug ("enabling voice\n");
+            if (poll_mode && alsa_poll_in (hw)) {
+                poll_mode = 0;
+            }
+            hw->poll_mode = poll_mode;
+
+            return alsa_voice_ctl (alsa->handle, "capture", VOICE_CTL_START);
        }
-        hw->poll_mode = poll_mode;

-        alsa_voice_ctl(alsa->handle, "capture", VOICE_CTL_START);
-    } else {
+    case VOICE_DISABLE:
        ldebug ("disabling voice\n");
        if (hw->poll_mode) {
            hw->poll_mode = 0;
-            alsa_fini_poll(&alsa->pollhlp);
+            alsa_fini_poll (&alsa->pollhlp);
        }
-        alsa_voice_ctl(alsa->handle, "capture", VOICE_CTL_PAUSE);
+        return alsa_voice_ctl (alsa->handle, "capture", VOICE_CTL_PAUSE);
    }
+
+    return -1;
 }

 static void alsa_init_per_direction(AudiodevAlsaPerDirectionOptions *apdo)
@@ -905,13 +1072,15 @@ static void alsa_audio_fini (void *opaque)
 static struct audio_pcm_ops alsa_pcm_ops = {
    .init_out = alsa_init_out,
    .fini_out = alsa_fini_out,
+    .run_out  = alsa_run_out,
    .write    = alsa_write,
-    .enable_out = alsa_enable_out,
+    .ctl_out  = alsa_ctl_out,

    .init_in  = alsa_init_in,
    .fini_in  = alsa_fini_in,
+    .run_in   = alsa_run_in,
    .read     = alsa_read,
-    .enable_in = alsa_enable_in,
+    .ctl_in   = alsa_ctl_in,
 };

 static struct audio_driver alsa_audio_driver = {
@@ -27,7 +27,6 @@

 #include "qemu/queue.h"
 #include "qapi/qapi-types-audio.h"
-#include "hw/qdev-properties.h"

 typedef void (*audio_callback_fn) (void *opaque, int avail);

@@ -79,10 +78,8 @@ typedef struct SWVoiceOut SWVoiceOut;
 typedef struct CaptureVoiceOut CaptureVoiceOut;
 typedef struct SWVoiceIn SWVoiceIn;

-typedef struct AudioState AudioState;
 typedef struct QEMUSoundCard {
    char *name;
-    AudioState *state;
    QLIST_ENTRY (QEMUSoundCard) entries;
 } QEMUSoundCard;

@@ -95,8 +92,7 @@ void AUD_log (const char *cap, const char *fmt, ...) GCC_FMT_ATTR(2, 3);

 void AUD_register_card (const char *name, QEMUSoundCard *card);
 void AUD_remove_card (QEMUSoundCard *card);
-CaptureVoiceOut *AUD_add_capture(
-    AudioState *s,
+CaptureVoiceOut *AUD_add_capture (
    struct audsettings *as,
    struct audio_capture_ops *ops,
    void *opaque
@@ -113,7 +109,7 @@ SWVoiceOut *AUD_open_out (
    );

 void AUD_close_out (QEMUSoundCard *card, SWVoiceOut *sw);
-size_t AUD_write (SWVoiceOut *sw, void *pcm_buf, size_t size);
+int  AUD_write (SWVoiceOut *sw, void *pcm_buf, int size);
 int  AUD_get_buffer_size_out (SWVoiceOut *sw);
 void AUD_set_active_out (SWVoiceOut *sw, int on);
 int  AUD_is_active_out (SWVoiceOut *sw);
@@ -124,16 +120,6 @@ uint64_t AUD_get_elapsed_usec_out (SWVoiceOut *sw, QEMUAudioTimeStamp *ts);
 void AUD_set_volume_out (SWVoiceOut *sw, int mute, uint8_t lvol, uint8_t rvol);
 void AUD_set_volume_in (SWVoiceIn *sw, int mute, uint8_t lvol, uint8_t rvol);

-#define AUDIO_MAX_CHANNELS 16
-typedef struct Volume {
-    bool mute;
-    int channels;
-    uint8_t vol[AUDIO_MAX_CHANNELS];
-} Volume;
-
-void audio_set_volume_out(SWVoiceOut *sw, Volume *vol);
-void audio_set_volume_in(SWVoiceIn *sw, Volume *vol);
-
 SWVoiceIn *AUD_open_in (
    QEMUSoundCard *card,
    SWVoiceIn *sw,
@@ -144,7 +130,7 @@ SWVoiceIn *AUD_open_in (
    );

 void AUD_close_in (QEMUSoundCard *card, SWVoiceIn *sw);
-size_t AUD_read (SWVoiceIn *sw, void *pcm_buf, size_t size);
+int  AUD_read (SWVoiceIn *sw, void *pcm_buf, int size);
 void AUD_set_active_in (SWVoiceIn *sw, int on);
 int  AUD_is_active_in (SWVoiceIn *sw);

@@ -157,8 +143,25 @@ static inline void *advance (void *p, int incr)
    return (d + incr);
 }

-int wav_start_capture(AudioState *state, CaptureState *s, const char *path,
-                      int freq, int bits, int nchannels);
+#ifdef __GNUC__
+#define audio_MIN(a, b) ( __extension__ ({      \
+    __typeof (a) ta = a;                        \
+    __typeof (b) tb = b;                        \
+    ((ta)>(tb)?(tb):(ta));                      \
+}))
+
+#define audio_MAX(a, b) ( __extension__ ({      \
+    __typeof (a) ta = a;                        \
+    __typeof (b) tb = b;                        \
+    ((ta)<(tb)?(tb):(ta));                      \
+}))
+#else
+#define audio_MIN(a, b) ((a)>(b)?(b):(a))
+#define audio_MAX(a, b) ((a)<(b)?(b):(a))
+#endif
+
+int wav_start_capture (CaptureState *s, const char *path, int freq,
+                       int bits, int nchannels);

 bool audio_is_cleaning_up(void);
 void audio_cleanup(void);
@@ -172,10 +175,4 @@ void audio_parse_option(const char *opt);
 void audio_init_audiodevs(void);
 void audio_legacy_help(void);

-AudioState *audio_state_by_name(const char *name);
-const char *audio_get_id(QEMUSoundCard *card);
-
-#define DEFINE_AUDIO_PROPERTIES(_s, _f)         \
-    DEFINE_PROP_AUDIODEV("audiodev", _s, _f)
-
 #endif /* QEMU_AUDIO_H */
@@ -43,70 +43,63 @@ struct audio_pcm_info {
    int sign;
    int freq;
    int nchannels;
-    int bytes_per_frame;
+    int align;
+    int shift;
    int bytes_per_second;
    int swap_endianness;
 };

-typedef struct AudioState AudioState;
 typedef struct SWVoiceCap SWVoiceCap;

-typedef struct STSampleBuffer {
-    size_t pos, size;
-    st_sample samples[];
-} STSampleBuffer;
-
 typedef struct HWVoiceOut {
-    AudioState *s;
    int enabled;
    int poll_mode;
    int pending_disable;
    struct audio_pcm_info info;

    f_sample *clip;
+
+    int rpos;
    uint64_t ts_helper;

-    STSampleBuffer *mix_buf;
-    void *buf_emul;
-    size_t pos_emul, pending_emul, size_emul;
+    struct st_sample *mix_buf;

-    size_t samples;
+    int samples;
    QLIST_HEAD (sw_out_listhead, SWVoiceOut) sw_head;
    QLIST_HEAD (sw_cap_listhead, SWVoiceCap) cap_head;
+    int ctl_caps;
    struct audio_pcm_ops *pcm_ops;
    QLIST_ENTRY (HWVoiceOut) entries;
 } HWVoiceOut;

 typedef struct HWVoiceIn {
-    AudioState *s;
    int enabled;
    int poll_mode;
    struct audio_pcm_info info;

    t_sample *conv;

-    size_t total_samples_captured;
+    int wpos;
+    int total_samples_captured;
    uint64_t ts_helper;

-    STSampleBuffer *conv_buf;
-    void *buf_emul;
-    size_t pos_emul, pending_emul, size_emul;
+    struct st_sample *conv_buf;

-    size_t samples;
+    int samples;
    QLIST_HEAD (sw_in_listhead, SWVoiceIn) sw_head;
+    int ctl_caps;
    struct audio_pcm_ops *pcm_ops;
    QLIST_ENTRY (HWVoiceIn) entries;
 } HWVoiceIn;

 struct SWVoiceOut {
    QEMUSoundCard *card;
-    AudioState *s;
    struct audio_pcm_info info;
    t_sample *conv;
    int64_t ratio;
    struct st_sample *buf;
    void *rate;
-    size_t total_hw_samples_mixed;
+    int total_hw_samples_mixed;
    int active;
    int empty;
    HWVoiceOut *hw;
@@ -118,12 +111,11 @@ struct SWVoiceOut {

 struct SWVoiceIn {
    QEMUSoundCard *card;
-    AudioState *s;
    int active;
    struct audio_pcm_info info;
    int64_t ratio;
    void *rate;
-    size_t total_hw_samples_acquired;
+    int total_hw_samples_acquired;
    struct st_sample *buf;
    f_sample *clip;
    HWVoiceIn *hw;
@@ -145,46 +137,24 @@ struct audio_driver {
    int max_voices_in;
    int voice_size_out;
    int voice_size_in;
+    int ctl_caps;
    QLIST_ENTRY(audio_driver) next;
 };

 struct audio_pcm_ops {
-    int    (*init_out)(HWVoiceOut *hw, audsettings *as, void *drv_opaque);
-    void   (*fini_out)(HWVoiceOut *hw);
-    size_t (*write)   (HWVoiceOut *hw, void *buf, size_t size);
-    /*
-     * get a buffer that after later can be passed to put_buffer_out; optional
-     * returns the buffer, and writes it's size to size (in bytes)
-     * this is unrelated to the above buffer_size_out function
-     */
-    void  *(*get_buffer_out)(HWVoiceOut *hw, size_t *size);
-    /*
-     * put back the buffer returned by get_buffer_out; optional
-     * buf must be equal the pointer returned by get_buffer_out,
-     * size may be smaller
-     */
-    size_t (*put_buffer_out)(HWVoiceOut *hw, void *buf, size_t size);
-    void   (*enable_out)(HWVoiceOut *hw, bool enable);
-    void   (*volume_out)(HWVoiceOut *hw, Volume *vol);
+    int  (*init_out)(HWVoiceOut *hw, struct audsettings *as, void *drv_opaque);
+    void (*fini_out)(HWVoiceOut *hw);
+    int  (*run_out) (HWVoiceOut *hw, int live);
+    int  (*write)   (SWVoiceOut *sw, void *buf, int size);
+    int  (*ctl_out) (HWVoiceOut *hw, int cmd, ...);

-    int    (*init_in) (HWVoiceIn *hw, audsettings *as, void *drv_opaque);
-    void   (*fini_in) (HWVoiceIn *hw);
-    size_t (*read)    (HWVoiceIn *hw, void *buf, size_t size);
-    void  *(*get_buffer_in)(HWVoiceIn *hw, size_t *size);
-    void   (*put_buffer_in)(HWVoiceIn *hw, void *buf, size_t size);
-    void   (*enable_in)(HWVoiceIn *hw, bool enable);
-    void   (*volume_in)(HWVoiceIn *hw, Volume *vol);
+    int  (*init_in) (HWVoiceIn *hw, struct audsettings *as, void *drv_opaque);
+    void (*fini_in) (HWVoiceIn *hw);
+    int  (*run_in)  (HWVoiceIn *hw);
+    int  (*read)    (SWVoiceIn *sw, void *buf, int size);
+    int  (*ctl_in)  (HWVoiceIn *hw, int cmd, ...);
 };

-void *audio_generic_get_buffer_in(HWVoiceIn *hw, size_t *size);
-void audio_generic_put_buffer_in(HWVoiceIn *hw, void *buf, size_t size);
-void *audio_generic_get_buffer_out(HWVoiceOut *hw, size_t *size);
-size_t audio_generic_put_buffer_out(HWVoiceOut *hw, void *buf, size_t size);
-size_t audio_generic_put_buffer_out_nowrite(HWVoiceOut *hw, void *buf,
-                                            size_t size);
-size_t audio_generic_write(HWVoiceOut *hw, void *buf, size_t size);
-size_t audio_generic_read(HWVoiceIn *hw, void *buf, size_t size);
-
 struct capture_callback {
    struct audio_capture_ops ops;
    void *opaque;
@@ -218,11 +188,6 @@ typedef struct AudioState {
    int nb_hw_voices_in;
    int vm_running;
    int64_t period_ticks;
-
-    bool timer_running;
-    uint64_t timer_last;
-
-    QTAILQ_ENTRY(AudioState) list;
 } AudioState;

 extern const struct mixeng_volume nominal_volume;
@@ -235,21 +200,26 @@ audio_driver *audio_driver_lookup(const char *name);
 void audio_pcm_init_info (struct audio_pcm_info *info, struct audsettings *as);
 void audio_pcm_info_clear_buf (struct audio_pcm_info *info, void *buf, int len);

+int  audio_pcm_sw_write (SWVoiceOut *sw, void *buf, int len);
+int  audio_pcm_hw_get_live_in (HWVoiceIn *hw);
+
+int  audio_pcm_sw_read (SWVoiceIn *sw, void *buf, int len);
+
+int audio_pcm_hw_clip_out (HWVoiceOut *hw, void *pcm_buf,
+                           int live, int pending);
+
 int audio_bug (const char *funcname, int cond);
 void *audio_calloc (const char *funcname, int nmemb, size_t size);

-void audio_run(AudioState *s, const char *msg);
+void audio_run (const char *msg);

-typedef struct RateCtl {
-    int64_t start_ticks;
-    int64_t bytes_sent;
-} RateCtl;
+#define VOICE_ENABLE 1
+#define VOICE_DISABLE 2
+#define VOICE_VOLUME 3

-void audio_rate_start(RateCtl *rate);
-size_t audio_rate_get_bytes(struct audio_pcm_info *info, RateCtl *rate,
-                            size_t bytes_avail);
+#define VOICE_VOLUME_CAP (1 << VOICE_VOLUME)

-static inline size_t audio_ring_dist(size_t dst, size_t src, size_t len)
+static inline int audio_ring_dist (int dst, int src, int len)
 {
    return (dst >= src) ? (dst - src) : (len - src + dst);
 }
@@ -0,0 +1,173 @@
+#include "qemu/osdep.h"
+#include "audio.h"
+
+#define AUDIO_CAP "audio-pt"
+
+#include "audio_int.h"
+#include "audio_pt_int.h"
+
+static void GCC_FMT_ATTR(3, 4) logerr (struct audio_pt *pt, int err,
+                                       const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start (ap, fmt);
+    AUD_vlog (pt->drv, fmt, ap);
+    va_end (ap);
+
+    AUD_log (NULL, "\n");
+    AUD_log (pt->drv, "Reason: %s\n", strerror (err));
+}
+
+int audio_pt_init (struct audio_pt *p, void *(*func) (void *),
+                   void *opaque, const char *drv, const char *cap)
+{
+    int err, err2;
+    const char *efunc;
+    sigset_t set, old_set;
+
+    p->drv = drv;
+
+    err = sigfillset (&set);
+    if (err) {
+        logerr(p, errno, "%s(%s): sigfillset failed", cap, __func__);
+        return -1;
+    }
+
+    err = pthread_mutex_init (&p->mutex, NULL);
+    if (err) {
+        efunc = "pthread_mutex_init";
+        goto err0;
+    }
+
+    err = pthread_cond_init (&p->cond, NULL);
+    if (err) {
+        efunc = "pthread_cond_init";
+        goto err1;
+    }
+
+    err = pthread_sigmask (SIG_BLOCK, &set, &old_set);
+    if (err) {
+        efunc = "pthread_sigmask";
+        goto err2;
+    }
+
+    err = pthread_create (&p->thread, NULL, func, opaque);
+
+    err2 = pthread_sigmask (SIG_SETMASK, &old_set, NULL);
+    if (err2) {
+        logerr(p, err2, "%s(%s): pthread_sigmask (restore) failed",
+               cap, __func__);
+        /* We have failed to restore original signal mask, all bets are off,
+           so terminate the process */
+        exit (EXIT_FAILURE);
+    }
+
+    if (err) {
+        efunc = "pthread_create";
+        goto err2;
+    }
+
+    return 0;
+
+ err2:
+    err2 = pthread_cond_destroy (&p->cond);
+    if (err2) {
+        logerr(p, err2, "%s(%s): pthread_cond_destroy failed", cap, __func__);
+    }
+
+ err1:
+    err2 = pthread_mutex_destroy (&p->mutex);
+    if (err2) {
+        logerr(p, err2, "%s(%s): pthread_mutex_destroy failed", cap, __func__);
+    }
+
+ err0:
+    logerr(p, err, "%s(%s): %s failed", cap, __func__, efunc);
+    return -1;
+}
+
+int audio_pt_fini (struct audio_pt *p, const char *cap)
+{
+    int err, ret = 0;
+
+    err = pthread_cond_destroy (&p->cond);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_cond_destroy failed", cap, __func__);
+        ret = -1;
+    }
+
+    err = pthread_mutex_destroy (&p->mutex);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_mutex_destroy failed", cap, __func__);
+        ret = -1;
+    }
+    return ret;
+}
+
+int audio_pt_lock (struct audio_pt *p, const char *cap)
+{
+    int err;
+
+    err = pthread_mutex_lock (&p->mutex);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_mutex_lock failed", cap, __func__);
+        return -1;
+    }
+    return 0;
+}
+
+int audio_pt_unlock (struct audio_pt *p, const char *cap)
+{
+    int err;
+
+    err = pthread_mutex_unlock (&p->mutex);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_mutex_unlock failed", cap, __func__);
+        return -1;
+    }
+    return 0;
+}
+
+int audio_pt_wait (struct audio_pt *p, const char *cap)
+{
+    int err;
+
+    err = pthread_cond_wait (&p->cond, &p->mutex);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_cond_wait failed", cap, __func__);
+        return -1;
+    }
+    return 0;
+}
+
+int audio_pt_unlock_and_signal (struct audio_pt *p, const char *cap)
+{
+    int err;
+
+    err = pthread_mutex_unlock (&p->mutex);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_mutex_unlock failed", cap, __func__);
+        return -1;
+    }
+    err = pthread_cond_signal (&p->cond);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_cond_signal failed", cap, __func__);
+        return -1;
+    }
+    return 0;
+}
+
+int audio_pt_join (struct audio_pt *p, void **arg, const char *cap)
+{
+    int err;
+    void *ret;
+
+    err = pthread_join (p->thread, &ret);
+    if (err) {
+        logerr(p, err, "%s(%s): pthread_join failed", cap, __func__);
+        return -1;
+    }
+    *arg = ret;
+    return 0;
+}
@@ -0,0 +1,22 @@
+#ifndef QEMU_AUDIO_PT_INT_H
+#define QEMU_AUDIO_PT_INT_H
+
+#include <pthread.h>
+
+struct audio_pt {
+    const char *drv;
+    pthread_t thread;
+    pthread_cond_t cond;
+    pthread_mutex_t mutex;
+};
+
+int audio_pt_init (struct audio_pt *, void *(*) (void *), void *,
+                   const char *, const char *);
+int audio_pt_fini (struct audio_pt *, const char *);
+int audio_pt_lock (struct audio_pt *, const char *);
+int audio_pt_unlock (struct audio_pt *, const char *);
+int audio_pt_wait (struct audio_pt *, const char *);
+int audio_pt_unlock_and_signal (struct audio_pt *, const char *);
+int audio_pt_join (struct audio_pt *, void **, const char *);
+
+#endif /* QEMU_AUDIO_PT_INT_H */
@@ -36,9 +36,9 @@
 #define HWBUF hw->conv_buf
 #endif

-static void glue(audio_init_nb_voices_, TYPE)(AudioState *s,
-                                              struct audio_driver *drv)
+static void glue (audio_init_nb_voices_, TYPE) (struct audio_driver *drv)
 {
+    AudioState *s = &glob_audio_state;
    int max_voices = glue (drv->max_voices_, TYPE);
    int voice_size = glue (drv->voice_size_, TYPE);

@@ -71,24 +71,20 @@ static void glue(audio_init_nb_voices_, TYPE)(AudioState *s,

 static void glue (audio_pcm_hw_free_resources_, TYPE) (HW *hw)
 {
-    g_free(hw->buf_emul);
    g_free (HWBUF);
    HWBUF = NULL;
 }

-static void glue(audio_pcm_hw_alloc_resources_, TYPE)(HW *hw)
+static int glue (audio_pcm_hw_alloc_resources_, TYPE) (HW *hw)
 {
-    if (glue(audio_get_pdo_, TYPE)(hw->s->dev)->mixing_engine) {
-        size_t samples = hw->samples;
-        if (audio_bug(__func__, samples == 0)) {
-            dolog("Attempted to allocate empty buffer\n");
-        }
-
-        HWBUF = g_malloc0(sizeof(STSampleBuffer) + sizeof(st_sample) * samples);
-        HWBUF->size = samples;
-    } else {
-        HWBUF = NULL;
+    HWBUF = audio_calloc(__func__, hw->samples, sizeof(struct st_sample));
+    if (!HWBUF) {
+        dolog ("Could not allocate " NAME " buffer (%d samples)\n",
+               hw->samples);
+        return -1;
    }
+
+    return 0;
 }

 static void glue (audio_pcm_sw_free_resources_, TYPE) (SW *sw)
@@ -107,11 +103,7 @@ static int glue (audio_pcm_sw_alloc_resources_, TYPE) (SW *sw)
 {
    int samples;

-    if (!glue(audio_get_pdo_, TYPE)(sw->s->dev)->mixing_engine) {
-        return 0;
-    }
-
-    samples = ((int64_t) sw->HWBUF->size << 32) / sw->ratio;
+    samples = ((int64_t) sw->hw->samples << 32) / sw->ratio;

    sw->buf = audio_calloc(__func__, samples, sizeof(struct st_sample));
    if (!sw->buf) {
@@ -191,8 +183,8 @@ static void glue (audio_pcm_hw_del_sw_, TYPE) (SW *sw)

 static void glue (audio_pcm_hw_gc_, TYPE) (HW **hwp)
 {
+    AudioState *s = &glob_audio_state;
    HW *hw = *hwp;
-    AudioState *s = hw->s;

    if (!hw->sw_head.lh_first) {
 #ifdef DAC
@@ -207,14 +199,15 @@ static void glue (audio_pcm_hw_gc_, TYPE) (HW **hwp)
    }
 }

-static HW *glue(audio_pcm_hw_find_any_, TYPE)(AudioState *s, HW *hw)
+static HW *glue (audio_pcm_hw_find_any_, TYPE) (HW *hw)
 {
+    AudioState *s = &glob_audio_state;
    return hw ? hw->entries.le_next : glue (s->hw_head_, TYPE).lh_first;
 }

-static HW *glue(audio_pcm_hw_find_any_enabled_, TYPE)(AudioState *s, HW *hw)
+static HW *glue (audio_pcm_hw_find_any_enabled_, TYPE) (HW *hw)
 {
-    while ((hw = glue(audio_pcm_hw_find_any_, TYPE)(s, hw))) {
+    while ((hw = glue (audio_pcm_hw_find_any_, TYPE) (hw))) {
        if (hw->enabled) {
            return hw;
        }
@@ -222,10 +215,12 @@ static HW *glue(audio_pcm_hw_find_any_enabled_, TYPE)(AudioState *s, HW *hw)
    return NULL;
 }

-static HW *glue(audio_pcm_hw_find_specific_, TYPE)(AudioState *s, HW *hw,
-                                                   struct audsettings *as)
+static HW *glue (audio_pcm_hw_find_specific_, TYPE) (
+    HW *hw,
+    struct audsettings *as
+    )
 {
-    while ((hw = glue(audio_pcm_hw_find_any_, TYPE)(s, hw))) {
+    while ((hw = glue (audio_pcm_hw_find_any_, TYPE) (hw))) {
        if (audio_pcm_info_eq (&hw->info, as)) {
            return hw;
        }
@@ -233,10 +228,10 @@ static HW *glue(audio_pcm_hw_find_specific_, TYPE)(AudioState *s, HW *hw,
    return NULL;
 }

-static HW *glue(audio_pcm_hw_add_new_, TYPE)(AudioState *s,
-                                             struct audsettings *as)
+static HW *glue (audio_pcm_hw_add_new_, TYPE) (struct audsettings *as)
 {
    HW *hw;
+    AudioState *s = &glob_audio_state;
    struct audio_driver *drv = s->drv;

    if (!glue (s->nb_hw_voices_, TYPE)) {
@@ -260,8 +255,8 @@ static HW *glue(audio_pcm_hw_add_new_, TYPE)(AudioState *s,
        return NULL;
    }

-    hw->s = s;
    hw->pcm_ops = drv->pcm_ops;
+    hw->ctl_caps = drv->ctl_caps;

    QLIST_INIT (&hw->sw_head);
 #ifdef DAC
@@ -272,7 +267,7 @@ static HW *glue(audio_pcm_hw_add_new_, TYPE)(AudioState *s,
    }

    if (audio_bug(__func__, hw->samples <= 0)) {
-        dolog("hw->samples=%zd\n", hw->samples);
+        dolog ("hw->samples=%d\n", hw->samples);
        goto err1;
    }

@@ -286,7 +281,9 @@ static HW *glue(audio_pcm_hw_add_new_, TYPE)(AudioState *s,
        [hw->info.swap_endianness]
        [audio_bits_to_index (hw->info.bits)];

-    glue(audio_pcm_hw_alloc_resources_, TYPE)(hw);
+    if (glue (audio_pcm_hw_alloc_resources_, TYPE) (hw)) {
+        goto err1;
+    }

    QLIST_INSERT_HEAD (&s->glue (hw_head_, TYPE), hw, entries);
    glue (s->nb_hw_voices_, TYPE) -= 1;
@@ -331,33 +328,33 @@ AudiodevPerDirectionOptions *glue(audio_get_pdo_, TYPE)(Audiodev *dev)
    abort();
 }

-static HW *glue(audio_pcm_hw_add_, TYPE)(AudioState *s, struct audsettings *as)
+static HW *glue (audio_pcm_hw_add_, TYPE) (struct audsettings *as)
 {
    HW *hw;
+    AudioState *s = &glob_audio_state;
    AudiodevPerDirectionOptions *pdo = glue(audio_get_pdo_, TYPE)(s->dev);

-    if (!pdo->mixing_engine || pdo->fixed_settings) {
-        hw = glue(audio_pcm_hw_add_new_, TYPE)(s, as);
-        if (!pdo->mixing_engine || hw) {
+    if (pdo->fixed_settings) {
+        hw = glue (audio_pcm_hw_add_new_, TYPE) (as);
+        if (hw) {
            return hw;
        }
    }

-    hw = glue(audio_pcm_hw_find_specific_, TYPE)(s, NULL, as);
+    hw = glue (audio_pcm_hw_find_specific_, TYPE) (NULL, as);
    if (hw) {
        return hw;
    }

-    hw = glue(audio_pcm_hw_add_new_, TYPE)(s, as);
+    hw = glue (audio_pcm_hw_add_new_, TYPE) (as);
    if (hw) {
        return hw;
    }

-    return glue(audio_pcm_hw_find_any_, TYPE)(s, NULL);
+    return glue (audio_pcm_hw_find_any_, TYPE) (NULL);
 }

-static SW *glue(audio_pcm_create_voice_pair_, TYPE)(
-    AudioState *s,
+static SW *glue (audio_pcm_create_voice_pair_, TYPE) (
    const char *sw_name,
    struct audsettings *as
    )
@@ -365,6 +362,7 @@ static SW *glue(audio_pcm_create_voice_pair_, TYPE)(
    SW *sw;
    HW *hw;
    struct audsettings hw_as;
+    AudioState *s = &glob_audio_state;
    AudiodevPerDirectionOptions *pdo = glue(audio_get_pdo_, TYPE)(s->dev);

    if (pdo->fixed_settings) {
@@ -380,9 +378,8 @@ static SW *glue(audio_pcm_create_voice_pair_, TYPE)(
               sw_name ? sw_name : "unknown", sizeof (*sw));
        goto err1;
    }
-    sw->s = s;

-    hw = glue(audio_pcm_hw_add_, TYPE)(s, &hw_as);
+    hw = glue (audio_pcm_hw_add_, TYPE) (&hw_as);
    if (!hw) {
        goto err2;
    }
@@ -433,8 +430,8 @@ SW *glue (AUD_open_, TYPE) (
    struct audsettings *as
    )
 {
-    AudioState *s;
-    AudiodevPerDirectionOptions *pdo;
+    AudioState *s = &glob_audio_state;
+    AudiodevPerDirectionOptions *pdo = glue(audio_get_pdo_, TYPE)(s->dev);

    if (audio_bug(__func__, !card || !name || !callback_fn || !as)) {
        dolog ("card=%p name=%p callback_fn=%p as=%p\n",
@@ -442,9 +439,6 @@ SW *glue (AUD_open_, TYPE) (
        goto fail;
    }

-    s = card->state;
-    pdo = glue(audio_get_pdo_, TYPE)(s->dev);
-
    ldebug ("open %s, freq %d, nchannels %d, fmt %d\n",
            name, as->freq, as->nchannels, as->fmt);

@@ -482,7 +476,7 @@ SW *glue (AUD_open_, TYPE) (
        }
    }
    else {
-        sw = glue(audio_pcm_create_voice_pair_, TYPE)(s, name, as);
+        sw = glue (audio_pcm_create_voice_pair_, TYPE) (name, as);
        if (!sw) {
            dolog ("Failed to create voice `%s'\n", name);
            return NULL;
@@ -43,6 +43,9 @@ typedef struct coreaudioVoiceOut {
    UInt32 audioDevicePropertyBufferFrameSize;
    AudioStreamBasicDescription outputStreamBasicDescription;
    AudioDeviceIOProcID ioprocid;
+    int live;
+    int decr;
+    int rpos;
 } coreaudioVoiceOut;

 #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
@@ -394,29 +397,31 @@ static int coreaudio_unlock (coreaudioVoiceOut *core, const char *fn_name)
    return 0;
 }

-#define COREAUDIO_WRAPPER_FUNC(name, ret_type, args_decl, args) \
-    static ret_type glue(coreaudio_, name)args_decl             \
-    {                                                           \
-        coreaudioVoiceOut *core = (coreaudioVoiceOut *) hw;     \
-        ret_type ret;                                           \
-                                                                \
-        if (coreaudio_lock(core, "coreaudio_" #name)) {         \
-            return 0;                                           \
-        }                                                       \
-                                                                \
-        ret = glue(audio_generic_, name)args;                   \
-                                                                \
-        coreaudio_unlock(core, "coreaudio_" #name);             \
-        return ret;                                             \
+static int coreaudio_run_out (HWVoiceOut *hw, int live)
+{
+    int decr;
+    coreaudioVoiceOut *core = (coreaudioVoiceOut *) hw;
+
+    if (coreaudio_lock (core, "coreaudio_run_out")) {
+        return 0;
    }
-COREAUDIO_WRAPPER_FUNC(get_buffer_out, void *, (HWVoiceOut *hw, size_t *size),
-                       (hw, size))
-COREAUDIO_WRAPPER_FUNC(put_buffer_out_nowrite, size_t,
-                       (HWVoiceOut *hw, void *buf, size_t size),
-                       (hw, buf, size))
-COREAUDIO_WRAPPER_FUNC(write, size_t, (HWVoiceOut *hw, void *buf, size_t size),
-                       (hw, buf, size))
-#undef COREAUDIO_WRAPPER_FUNC
+
+    if (core->decr > live) {
+        ldebug ("core->decr %d live %d core->live %d\n",
+                core->decr,
+                live,
+                core->live);
+    }
+
+    decr = audio_MIN (core->decr, live);
+    core->decr -= decr;
+
+    core->live = live - decr;
+    hw->rpos = core->rpos;
+
+    coreaudio_unlock (core, "coreaudio_run_out");
+    return decr;
+}

 /* callback to feed audiooutput buffer */
 static OSStatus audioDeviceIOProc(
@@ -428,11 +433,19 @@ static OSStatus audioDeviceIOProc(
    const AudioTimeStamp* inOutputTime,
    void* hwptr)
 {
-    UInt32 frameCount, pending_frames;
-    void *out = outOutputData->mBuffers[0].mData;
+    UInt32 frame, frameCount;
+    float *out = outOutputData->mBuffers[0].mData;
    HWVoiceOut *hw = hwptr;
    coreaudioVoiceOut *core = (coreaudioVoiceOut *) hwptr;
-    size_t len;
+    int rpos, live;
+    struct st_sample *src;
+#ifndef FLOAT_MIXENG
+#ifdef RECIPROCAL
+    const float scale = 1.f / UINT_MAX;
+#else
+    const float scale = UINT_MAX;
+#endif
+#endif

    if (coreaudio_lock (core, "audioDeviceIOProc")) {
        inInputTime = 0;
@@ -440,49 +453,45 @@ static OSStatus audioDeviceIOProc(
    }

    frameCount = core->audioDevicePropertyBufferFrameSize;
-    pending_frames = hw->pending_emul / hw->info.bytes_per_frame;
+    live = core->live;

    /* if there are not enough samples, set signal and return */
-    if (pending_frames < frameCount) {
+    if (live < frameCount) {
        inInputTime = 0;
        coreaudio_unlock (core, "audioDeviceIOProc(empty)");
        return 0;
    }

-    len = frameCount * hw->info.bytes_per_frame;
-    while (len) {
-        size_t write_len;
-        ssize_t start = ((ssize_t) hw->pos_emul) - hw->pending_emul;
-        if (start < 0) {
-            start += hw->size_emul;
-        }
-        assert(start >= 0 && start < hw->size_emul);
+    rpos = core->rpos;
+    src = hw->mix_buf + rpos;

-        write_len = MIN(MIN(hw->pending_emul, len),
-                        hw->size_emul - start);
-
-        memcpy(out, hw->buf_emul + start, write_len);
-        hw->pending_emul -= write_len;
-        len -= write_len;
-        out += write_len;
+    /* fill buffer */
+    for (frame = 0; frame < frameCount; frame++) {
+#ifdef FLOAT_MIXENG
+        *out++ = src[frame].l; /* left channel */
+        *out++ = src[frame].r; /* right channel */
+#else
+#ifdef RECIPROCAL
+        *out++ = src[frame].l * scale; /* left channel */
+        *out++ = src[frame].r * scale; /* right channel */
+#else
+        *out++ = src[frame].l / scale; /* left channel */
+        *out++ = src[frame].r / scale; /* right channel */
+#endif
+#endif
    }

+    rpos = (rpos + frameCount) % hw->samples;
+    core->decr += frameCount;
+    core->rpos = rpos;
+
    coreaudio_unlock (core, "audioDeviceIOProc");
    return 0;
 }

-static UInt32 coreaudio_get_flags(struct audio_pcm_info *info,
-                                  struct audsettings *as)
+static int coreaudio_write (SWVoiceOut *sw, void *buf, int len)
 {
-    UInt32 flags = info->sign ? kAudioFormatFlagIsSignedInteger : 0;
-    if (as->endianness) { /* 0 = little, 1 = big */
-        flags |= kAudioFormatFlagIsBigEndian;
-    }
-
-    if (flags == 0) { /* must not be 0 */
-        flags = kAudioFormatFlagsAreAllClear;
-    }
-    return flags;
+    return audio_pcm_sw_write (sw, buf, len);
 }

 static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
@@ -572,16 +581,6 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,

    /* set Samplerate */
    core->outputStreamBasicDescription.mSampleRate = (Float64) as->freq;
-    core->outputStreamBasicDescription.mFormatID = kAudioFormatLinearPCM;
-    core->outputStreamBasicDescription.mFormatFlags =
-        coreaudio_get_flags(&hw->info, as);
-    core->outputStreamBasicDescription.mBytesPerPacket =
-        core->outputStreamBasicDescription.mBytesPerFrame =
-        hw->info.nchannels * hw->info.bits / 8;
-    core->outputStreamBasicDescription.mFramesPerPacket = 1;
-    core->outputStreamBasicDescription.mChannelsPerFrame = hw->info.nchannels;
-    core->outputStreamBasicDescription.mBitsPerChannel = hw->info.bits;
-
    status = coreaudio_set_streamformat(core->outputDeviceID,
                                        &core->outputStreamBasicDescription);
    if (status != kAudioHardwareNoError) {
@@ -648,12 +647,13 @@ static void coreaudio_fini_out (HWVoiceOut *hw)
    }
 }

-static void coreaudio_enable_out(HWVoiceOut *hw, bool enable)
+static int coreaudio_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
    OSStatus status;
    coreaudioVoiceOut *core = (coreaudioVoiceOut *) hw;

-    if (enable) {
+    switch (cmd) {
+    case VOICE_ENABLE:
        /* start playback */
        if (!isPlaying(core->outputDeviceID)) {
            status = AudioDeviceStart(core->outputDeviceID, core->ioprocid);
@@ -661,7 +661,9 @@ static void coreaudio_enable_out(HWVoiceOut *hw, bool enable)
                coreaudio_logerr (status, "Could not resume playback\n");
            }
        }
-    } else {
+        break;
+
+    case VOICE_DISABLE:
        /* stop playback */
        if (!audio_is_cleaning_up()) {
            if (isPlaying(core->outputDeviceID)) {
@@ -672,7 +674,9 @@ static void coreaudio_enable_out(HWVoiceOut *hw, bool enable)
                }
            }
        }
+        break;
    }
+    return 0;
 }

 static void *coreaudio_audio_init(Audiodev *dev)
@@ -687,10 +691,9 @@ static void coreaudio_audio_fini (void *opaque)
 static struct audio_pcm_ops coreaudio_pcm_ops = {
    .init_out = coreaudio_init_out,
    .fini_out = coreaudio_fini_out,
+    .run_out  = coreaudio_run_out,
    .write    = coreaudio_write,
-    .get_buffer_out = coreaudio_get_buffer_out,
-    .put_buffer_out = coreaudio_put_buffer_out_nowrite,
-    .enable_out = coreaudio_enable_out
+    .ctl_out  = coreaudio_ctl_out
 };

 static struct audio_driver coreaudio_audio_driver = {
@@ -29,8 +29,6 @@
 #define BUFPTR LPDIRECTSOUNDCAPTUREBUFFER
 #define FIELD dsound_capture_buffer
 #define FIELD2 dsound_capture
-#define HWVOICE HWVoiceIn
-#define DSOUNDVOICE DSoundVoiceIn
 #else
 #define NAME "playback buffer"
 #define NAME2 "DirectSound"
@@ -39,8 +37,6 @@
 #define BUFPTR LPDIRECTSOUNDBUFFER
 #define FIELD dsound_buffer
 #define FIELD2 dsound
-#define HWVOICE HWVoiceOut
-#define DSOUNDVOICE DSoundVoiceOut
 #endif

 static int glue (dsound_unlock_, TYPE) (
@@ -76,6 +72,8 @@ static int glue (dsound_lock_, TYPE) (
    )
 {
    HRESULT hr;
+    LPVOID p1 = NULL, p2 = NULL;
+    DWORD blen1 = 0, blen2 = 0;
    DWORD flag;

 #ifdef DSBTYPE_IN
@@ -83,7 +81,7 @@ static int glue (dsound_lock_, TYPE) (
 #else
    flag = entire ? DSBLOCK_ENTIREBUFFER : 0;
 #endif
-    hr = glue(IFACE, _Lock)(buf, pos, len, p1p, blen1p, p2p, blen2p, flag);
+    hr = glue(IFACE, _Lock)(buf, pos, len, &p1, &blen1, &p2, &blen2, flag);

    if (FAILED (hr)) {
 #ifndef DSBTYPE_IN
@@ -98,34 +96,34 @@ static int glue (dsound_lock_, TYPE) (
        goto fail;
    }

-    if ((p1p && *p1p && (*blen1p % info->bytes_per_frame)) ||
-        (p2p && *p2p && (*blen2p % info->bytes_per_frame))) {
-        dolog("DirectSound returned misaligned buffer %ld %ld\n",
-              *blen1p, *blen2p);
-        glue(dsound_unlock_, TYPE)(buf, *p1p, p2p ? *p2p : NULL, *blen1p,
-                                   blen2p ? *blen2p : 0);
+    if ((p1 && (blen1 & info->align)) || (p2 && (blen2 & info->align))) {
+        dolog ("DirectSound returned misaligned buffer %ld %ld\n",
+               blen1, blen2);
+        glue (dsound_unlock_, TYPE) (buf, p1, p2, blen1, blen2);
        goto fail;
    }

-    if (p1p && !*p1p && *blen1p) {
-        dolog("warning: !p1 && blen1=%ld\n", *blen1p);
-        *blen1p = 0;
+    if (!p1 && blen1) {
+        dolog ("warning: !p1 && blen1=%ld\n", blen1);
+        blen1 = 0;
    }

-    if (p2p && !*p2p && *blen2p) {
-        dolog("warning: !p2 && blen2=%ld\n", *blen2p);
-        *blen2p = 0;
+    if (!p2 && blen2) {
+        dolog ("warning: !p2 && blen2=%ld\n", blen2);
+        blen2 = 0;
    }

+    *p1p = p1;
+    *p2p = p2;
+    *blen1p = blen1;
+    *blen2p = blen2;
    return 0;

 fail:
    *p1p = NULL - 1;
+    *p2p = NULL - 1;
    *blen1p = -1;
-    if (p2p) {
-        *p2p = NULL - 1;
-        *blen2p = -1;
-    }
+    *blen2p = -1;
    return -1;
 }

@@ -244,22 +242,25 @@ static int dsound_init_out(HWVoiceOut *hw, struct audsettings *as,
        goto fail0;
    }

+    ds->first_time = 1;
    obt_as.endianness = 0;
    audio_pcm_init_info (&hw->info, &obt_as);

-    if (bc.dwBufferBytes % hw->info.bytes_per_frame) {
+    if (bc.dwBufferBytes & hw->info.align) {
        dolog (
            "GetCaps returned misaligned buffer size %ld, alignment %d\n",
-            bc.dwBufferBytes, hw->info.bytes_per_frame
+            bc.dwBufferBytes, hw->info.align + 1
            );
    }
-    hw->size_emul = bc.dwBufferBytes;
-    hw->samples = bc.dwBufferBytes / hw->info.bytes_per_frame;
+    hw->samples = bc.dwBufferBytes >> hw->info.shift;
    ds->s = s;

 #ifdef DEBUG_DSOUND
    dolog ("caps %ld, desc %ld\n",
           bc.dwBufferBytes, bd.dwBufferBytes);
+
+    dolog ("bufsize %d, freq %d, chan %d, fmt %d\n",
+           hw->bufsize, settings.freq, settings.nchannels, settings.fmt);
 #endif
    return 0;

@@ -275,5 +276,3 @@ static int dsound_init_out(HWVoiceOut *hw, struct audsettings *as,
 #undef BUFPTR
 #undef FIELD
 #undef FIELD2
-#undef HWVOICE
-#undef DSOUNDVOICE
@@ -53,11 +53,19 @@ typedef struct {
 typedef struct {
    HWVoiceOut hw;
    LPDIRECTSOUNDBUFFER dsound_buffer;
+    DWORD old_pos;
+    int first_time;
    dsound *s;
+#ifdef DEBUG_DSOUND
+    DWORD old_ppos;
+    DWORD played;
+    DWORD mixed;
+#endif
 } DSoundVoiceOut;

 typedef struct {
    HWVoiceIn hw;
+    int first_time;
    LPDIRECTSOUNDCAPTUREBUFFER dsound_capture_buffer;
    dsound *s;
 } DSoundVoiceIn;
@@ -235,6 +243,11 @@ static void GCC_FMT_ATTR (3, 4) dsound_logerr2 (
    dsound_log_hresult (hr);
 }

+static uint64_t usecs_to_bytes(struct audio_pcm_info *info, uint32_t usecs)
+{
+    return muldiv64(usecs, info->bytes_per_second, 1000000);
+}
+
 #ifdef DEBUG_DSOUND
 static void print_wave_format (WAVEFORMATEX *wfx)
 {
@@ -299,6 +312,33 @@ static int dsound_get_status_in (LPDIRECTSOUNDCAPTUREBUFFER dscb,
    return 0;
 }

+static void dsound_write_sample (HWVoiceOut *hw, uint8_t *dst, int dst_len)
+{
+    int src_len1 = dst_len;
+    int src_len2 = 0;
+    int pos = hw->rpos + dst_len;
+    struct st_sample *src1 = hw->mix_buf + hw->rpos;
+    struct st_sample *src2 = NULL;
+
+    if (pos > hw->samples) {
+        src_len1 = hw->samples - hw->rpos;
+        src2 = hw->mix_buf;
+        src_len2 = dst_len - src_len1;
+        pos = src_len2;
+    }
+
+    if (src_len1) {
+        hw->clip (dst, src1, src_len1);
+    }
+
+    if (src_len2) {
+        dst = advance (dst, src_len1 << hw->info.shift);
+        hw->clip (dst, src2, src_len2);
+    }
+
+    hw->rpos = pos % hw->samples;
+}
+
 static void dsound_clear_sample (HWVoiceOut *hw, LPDIRECTSOUNDBUFFER dsb,
                                 dsound *s)
 {
@@ -310,7 +350,7 @@ static void dsound_clear_sample (HWVoiceOut *hw, LPDIRECTSOUNDBUFFER dsb,
        dsb,
        &hw->info,
        0,
-        hw->size_emul,
+        hw->samples << hw->info.shift,
        &p1, &p2,
        &blen1, &blen2,
        1,
@@ -320,8 +360,8 @@ static void dsound_clear_sample (HWVoiceOut *hw, LPDIRECTSOUNDBUFFER dsb,
        return;
    }

-    len1 = blen1 / hw->info.bytes_per_frame;
-    len2 = blen2 / hw->info.bytes_per_frame;
+    len1 = blen1 >> hw->info.shift;
+    len2 = blen2 >> hw->info.shift;

 #ifdef DEBUG_DSOUND
    dolog ("clear %p,%ld,%ld %p,%ld,%ld\n",
@@ -361,7 +401,7 @@ static int dsound_open (dsound *s)
    return 0;
 }

-static void dsound_enable_out(HWVoiceOut *hw, bool enable)
+static int dsound_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
    HRESULT hr;
    DWORD status;
@@ -371,17 +411,18 @@ static void dsound_enable_out(HWVoiceOut *hw, bool enable)

    if (!dsb) {
        dolog ("Attempt to control voice without a buffer\n");
-        return;
+        return 0;
    }

-    if (enable) {
+    switch (cmd) {
+    case VOICE_ENABLE:
        if (dsound_get_status_out (dsb, &status, s)) {
-            return;
+            return -1;
        }

        if (status & DSBSTATUS_PLAYING) {
            dolog ("warning: Voice is already playing\n");
-            return;
+            return 0;
        }

        dsound_clear_sample (hw, dsb, s);
@@ -389,74 +430,170 @@ static void dsound_enable_out(HWVoiceOut *hw, bool enable)
        hr = IDirectSoundBuffer_Play (dsb, 0, 0, DSBPLAY_LOOPING);
        if (FAILED (hr)) {
            dsound_logerr (hr, "Could not start playing buffer\n");
-            return;
+            return -1;
        }
-    } else {
+        break;
+
+    case VOICE_DISABLE:
        if (dsound_get_status_out (dsb, &status, s)) {
-            return;
+            return -1;
        }

        if (status & DSBSTATUS_PLAYING) {
            hr = IDirectSoundBuffer_Stop (dsb);
            if (FAILED (hr)) {
                dsound_logerr (hr, "Could not stop playing buffer\n");
-                return;
+                return -1;
            }
        }
        else {
            dolog ("warning: Voice is not playing\n");
        }
+        break;
    }
+    return 0;
 }

-static void *dsound_get_buffer_out(HWVoiceOut *hw, size_t *size)
+static int dsound_write (SWVoiceOut *sw, void *buf, int len)
+{
+    return audio_pcm_sw_write (sw, buf, len);
+}
+
+static int dsound_run_out (HWVoiceOut *hw, int live)
 {
-    DSoundVoiceOut *ds = (DSoundVoiceOut *) hw;
-    LPDIRECTSOUNDBUFFER dsb = ds->dsound_buffer;
-    HRESULT hr;
-    DWORD ppos, act_size;
-    size_t req_size;
    int err;
-    void *ret;
-
-    hr = IDirectSoundBuffer_GetCurrentPosition(dsb, &ppos, NULL);
-    if (FAILED(hr)) {
-        dsound_logerr(hr, "Could not get playback buffer position\n");
-        *size = 0;
-        return NULL;
-    }
-
-    req_size = audio_ring_dist(ppos, hw->pos_emul, hw->size_emul);
-    req_size = MIN(req_size, hw->size_emul - hw->pos_emul);
-
-    err = dsound_lock_out(dsb, &hw->info, hw->pos_emul, req_size, &ret, NULL,
-                          &act_size, NULL, false, ds->s);
-    if (err) {
-        dolog("Failed to lock buffer\n");
-        *size = 0;
-        return NULL;
-    }
-
-    *size = act_size;
-    return ret;
-}
-
-static size_t dsound_put_buffer_out(HWVoiceOut *hw, void *buf, size_t len)
-{
+    HRESULT hr;
    DSoundVoiceOut *ds = (DSoundVoiceOut *) hw;
    LPDIRECTSOUNDBUFFER dsb = ds->dsound_buffer;
-    int err = dsound_unlock_out(dsb, buf, NULL, len, 0);
+    int len, hwshift;
+    DWORD blen1, blen2;
+    DWORD len1, len2;
+    DWORD decr;
+    DWORD wpos, ppos, old_pos;
+    LPVOID p1, p2;
+    int bufsize;
+    dsound *s = ds->s;
+    AudiodevDsoundOptions *dso = &s->dev->u.dsound;

-    if (err) {
-        dolog("Failed to unlock buffer!!\n");
+    if (!dsb) {
+        dolog ("Attempt to run empty with playback buffer\n");
        return 0;
    }
-    hw->pos_emul = (hw->pos_emul + len) % hw->size_emul;

-    return len;
+    hwshift = hw->info.shift;
+    bufsize = hw->samples << hwshift;
+
+    hr = IDirectSoundBuffer_GetCurrentPosition (
+        dsb,
+        &ppos,
+        ds->first_time ? &wpos : NULL
+        );
+    if (FAILED (hr)) {
+        dsound_logerr (hr, "Could not get playback buffer position\n");
+        return 0;
+    }
+
+    len = live << hwshift;
+
+    if (ds->first_time) {
+        if (dso->latency) {
+            DWORD cur_blat;
+
+            cur_blat = audio_ring_dist (wpos, ppos, bufsize);
+            ds->first_time = 0;
+            old_pos = wpos;
+            old_pos +=
+                usecs_to_bytes(&hw->info, dso->latency) - cur_blat;
+            old_pos %= bufsize;
+            old_pos &= ~hw->info.align;
+        }
+        else {
+            old_pos = wpos;
+        }
+#ifdef DEBUG_DSOUND
+        ds->played = 0;
+        ds->mixed = 0;
+#endif
+    }
+    else {
+        if (ds->old_pos == ppos) {
+#ifdef DEBUG_DSOUND
+            dolog ("old_pos == ppos\n");
+#endif
+            return 0;
+        }
+
+#ifdef DEBUG_DSOUND
+        ds->played += audio_ring_dist (ds->old_pos, ppos, hw->bufsize);
+#endif
+        old_pos = ds->old_pos;
+    }
+
+    if ((old_pos < ppos) && ((old_pos + len) > ppos)) {
+        len = ppos - old_pos;
+    }
+    else {
+        if ((old_pos > ppos) && ((old_pos + len) > (ppos + bufsize))) {
+            len = bufsize - old_pos + ppos;
+        }
+    }
+
+    if (audio_bug(__func__, len < 0 || len > bufsize)) {
+        dolog ("len=%d bufsize=%d old_pos=%ld ppos=%ld\n",
+               len, bufsize, old_pos, ppos);
+        return 0;
+    }
+
+    len &= ~hw->info.align;
+    if (!len) {
+        return 0;
+    }
+
+#ifdef DEBUG_DSOUND
+    ds->old_ppos = ppos;
+#endif
+    err = dsound_lock_out (
+        dsb,
+        &hw->info,
+        old_pos,
+        len,
+        &p1, &p2,
+        &blen1, &blen2,
+        0,
+        s
+        );
+    if (err) {
+        return 0;
+    }
+
+    len1 = blen1 >> hwshift;
+    len2 = blen2 >> hwshift;
+    decr = len1 + len2;
+
+    if (p1 && len1) {
+        dsound_write_sample (hw, p1, len1);
+    }
+
+    if (p2 && len2) {
+        dsound_write_sample (hw, p2, len2);
+    }
+
+    dsound_unlock_out (dsb, p1, p2, blen1, blen2);
+    ds->old_pos = (old_pos + (decr << hwshift)) % bufsize;
+
+#ifdef DEBUG_DSOUND
+    ds->mixed += decr << hwshift;
+
+    dolog ("played %lu mixed %lu diff %ld sec %f\n",
+           ds->played,
+           ds->mixed,
+           ds->mixed - ds->played,
+           abs (ds->mixed - ds->played) / (double) hw->info.bytes_per_second);
+#endif
+    return decr;
 }

-static void dsound_enable_in(HWVoiceIn *hw, bool enable)
+static int dsound_ctl_in (HWVoiceIn *hw, int cmd, ...)
 {
    HRESULT hr;
    DWORD status;
@@ -465,17 +602,18 @@ static void dsound_enable_in(HWVoiceIn *hw, bool enable)

    if (!dscb) {
        dolog ("Attempt to control capture voice without a buffer\n");
-        return;
+        return -1;
    }

-    if (enable) {
+    switch (cmd) {
+    case VOICE_ENABLE:
        if (dsound_get_status_in (dscb, &status)) {
-            return;
+            return -1;
        }

        if (status & DSCBSTATUS_CAPTURING) {
            dolog ("warning: Voice is already capturing\n");
-            return;
+            return 0;
        }

        /* clear ?? */
@@ -483,69 +621,125 @@ static void dsound_enable_in(HWVoiceIn *hw, bool enable)
        hr = IDirectSoundCaptureBuffer_Start (dscb, DSCBSTART_LOOPING);
        if (FAILED (hr)) {
            dsound_logerr (hr, "Could not start capturing\n");
-            return;
+            return -1;
        }
-    } else {
+        break;
+
+    case VOICE_DISABLE:
        if (dsound_get_status_in (dscb, &status)) {
-            return;
+            return -1;
        }

        if (status & DSCBSTATUS_CAPTURING) {
            hr = IDirectSoundCaptureBuffer_Stop (dscb);
            if (FAILED (hr)) {
                dsound_logerr (hr, "Could not stop capturing\n");
-                return;
+                return -1;
            }
        }
        else {
            dolog ("warning: Voice is not capturing\n");
        }
+        break;
    }
+    return 0;
 }

-static void *dsound_get_buffer_in(HWVoiceIn *hw, size_t *size)
+static int dsound_read (SWVoiceIn *sw, void *buf, int len)
+{
+    return audio_pcm_sw_read (sw, buf, len);
+}
+
+static int dsound_run_in (HWVoiceIn *hw)
 {
-    DSoundVoiceIn *ds = (DSoundVoiceIn *) hw;
-    LPDIRECTSOUNDCAPTUREBUFFER dscb = ds->dsound_capture_buffer;
-    HRESULT hr;
-    DWORD cpos, act_size;
-    size_t req_size;
    int err;
-    void *ret;
-
-    hr = IDirectSoundCaptureBuffer_GetCurrentPosition(dscb, &cpos, NULL);
-    if (FAILED(hr)) {
-        dsound_logerr(hr, "Could not get capture buffer position\n");
-        *size = 0;
-        return NULL;
-    }
-
-    req_size = audio_ring_dist(cpos, hw->pos_emul, hw->size_emul);
-    req_size = MIN(req_size, hw->size_emul - hw->pos_emul);
-
-    err = dsound_lock_in(dscb, &hw->info, hw->pos_emul, req_size, &ret, NULL,
-                         &act_size, NULL, false, ds->s);
-    if (err) {
-        dolog("Failed to lock buffer\n");
-        *size = 0;
-        return NULL;
-    }
-
-    *size = act_size;
-    return ret;
-}
-
-static void dsound_put_buffer_in(HWVoiceIn *hw, void *buf, size_t len)
-{
+    HRESULT hr;
    DSoundVoiceIn *ds = (DSoundVoiceIn *) hw;
    LPDIRECTSOUNDCAPTUREBUFFER dscb = ds->dsound_capture_buffer;
-    int err = dsound_unlock_in(dscb, buf, NULL, len, 0);
+    int live, len, dead;
+    DWORD blen1, blen2;
+    DWORD len1, len2;
+    DWORD decr;
+    DWORD cpos, rpos;
+    LPVOID p1, p2;
+    int hwshift;
+    dsound *s = ds->s;

-    if (err) {
-        dolog("Failed to unlock buffer!!\n");
-        return;
+    if (!dscb) {
+        dolog ("Attempt to run without capture buffer\n");
+        return 0;
    }
-    hw->pos_emul = (hw->pos_emul + len) % hw->size_emul;
+
+    hwshift = hw->info.shift;
+
+    live = audio_pcm_hw_get_live_in (hw);
+    dead = hw->samples - live;
+    if (!dead) {
+        return 0;
+    }
+
+    hr = IDirectSoundCaptureBuffer_GetCurrentPosition (
+        dscb,
+        &cpos,
+        ds->first_time ? &rpos : NULL
+        );
+    if (FAILED (hr)) {
+        dsound_logerr (hr, "Could not get capture buffer position\n");
+        return 0;
+    }
+
+    if (ds->first_time) {
+        ds->first_time = 0;
+        if (rpos & hw->info.align) {
+            ldebug ("warning: Misaligned capture read position %ld(%d)\n",
+                    rpos, hw->info.align);
+        }
+        hw->wpos = rpos >> hwshift;
+    }
+
+    if (cpos & hw->info.align) {
+        ldebug ("warning: Misaligned capture position %ld(%d)\n",
+                cpos, hw->info.align);
+    }
+    cpos >>= hwshift;
+
+    len = audio_ring_dist (cpos, hw->wpos, hw->samples);
+    if (!len) {
+        return 0;
+    }
+    len = audio_MIN (len, dead);
+
+    err = dsound_lock_in (
+        dscb,
+        &hw->info,
+        hw->wpos << hwshift,
+        len << hwshift,
+        &p1,
+        &p2,
+        &blen1,
+        &blen2,
+        0,
+        s
+        );
+    if (err) {
+        return 0;
+    }
+
+    len1 = blen1 >> hwshift;
+    len2 = blen2 >> hwshift;
+    decr = len1 + len2;
+
+    if (p1 && len1) {
+        hw->conv (hw->conv_buf + hw->wpos, p1, len1);
+    }
+
+    if (p2 && len2) {
+        hw->conv (hw->conv_buf, p2, len2);
+    }
+
+    dsound_unlock_in (dscb, p1, p2, blen1, blen2);
+    hw->wpos = (hw->wpos + decr) % hw->samples;
+    return decr;
 }

 static void dsound_audio_fini (void *opaque)
@@ -661,17 +855,15 @@ static void *dsound_audio_init(Audiodev *dev)
 static struct audio_pcm_ops dsound_pcm_ops = {
    .init_out = dsound_init_out,
    .fini_out = dsound_fini_out,
-    .write    = audio_generic_write,
-    .get_buffer_out = dsound_get_buffer_out,
-    .put_buffer_out = dsound_put_buffer_out,
-    .enable_out = dsound_enable_out,
+    .run_out  = dsound_run_out,
+    .write    = dsound_write,
+    .ctl_out  = dsound_ctl_out,

    .init_in  = dsound_init_in,
    .fini_in  = dsound_fini_in,
-    .read     = audio_generic_read,
-    .get_buffer_in = dsound_get_buffer_in,
-    .put_buffer_in = dsound_put_buffer_in,
-    .enable_in = dsound_enable_in,
+    .run_in   = dsound_run_in,
+    .read     = dsound_read,
+    .ctl_in   = dsound_ctl_in
 };

 static struct audio_driver dsound_audio_driver = {
@@ -33,7 +33,6 @@ struct st_sample { mixeng_real l; mixeng_real r; };
 struct mixeng_volume { int mute; int64_t r; int64_t l; };
 struct st_sample { int64_t l; int64_t r; };
 #endif
-typedef struct st_sample st_sample;

 typedef void (t_sample) (struct st_sample *dst, const void *src, int samples);
 typedef void (f_sample) (void *dst, const struct st_sample *src, int samples);
@@ -42,10 +41,10 @@ extern t_sample *mixeng_conv[2][2][2][3];
 extern f_sample *mixeng_clip[2][2][2][3];

 void *st_rate_start (int inrate, int outrate);
-void st_rate_flow(void *opaque, st_sample *ibuf, st_sample *obuf,
-                  size_t *isamp, size_t *osamp);
-void st_rate_flow_mix(void *opaque, st_sample *ibuf, st_sample *obuf,
-                      size_t *isamp, size_t *osamp);
+void st_rate_flow (void *opaque, struct st_sample *ibuf, struct st_sample *obuf,
+                   int *isamp, int *osamp);
+void st_rate_flow_mix (void *opaque, struct st_sample *ibuf, struct st_sample *obuf,
+                       int *isamp, int *osamp);
 void st_rate_stop (void *opaque);
 void mixeng_clear (struct st_sample *buf, int len);
 void mixeng_volume (struct st_sample *buf, int len, struct mixeng_volume *vol);
@@ -33,27 +33,43 @@

 typedef struct NoVoiceOut {
    HWVoiceOut hw;
-    RateCtl rate;
+    int64_t old_ticks;
 } NoVoiceOut;

 typedef struct NoVoiceIn {
    HWVoiceIn hw;
-    RateCtl rate;
+    int64_t old_ticks;
 } NoVoiceIn;

-static size_t no_write(HWVoiceOut *hw, void *buf, size_t len)
+static int no_run_out (HWVoiceOut *hw, int live)
 {
    NoVoiceOut *no = (NoVoiceOut *) hw;
-    return audio_rate_get_bytes(&hw->info, &no->rate, len);
+    int decr, samples;
+    int64_t now;
+    int64_t ticks;
+    int64_t bytes;
+
+    now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    ticks = now - no->old_ticks;
+    bytes = muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
+    bytes = audio_MIN(bytes, INT_MAX);
+    samples = bytes >> hw->info.shift;
+
+    no->old_ticks = now;
+    decr = audio_MIN (live, samples);
+    hw->rpos = (hw->rpos + decr) % hw->samples;
+    return decr;
+}
+
+static int no_write (SWVoiceOut *sw, void *buf, int len)
+{
+    return audio_pcm_sw_write(sw, buf, len);
 }

 static int no_init_out(HWVoiceOut *hw, struct audsettings *as, void *drv_opaque)
 {
-    NoVoiceOut *no = (NoVoiceOut *) hw;
-
    audio_pcm_init_info (&hw->info, as);
    hw->samples = 1024;
-    audio_rate_start(&no->rate);
    return 0;
 }

@@ -62,22 +78,17 @@ static void no_fini_out (HWVoiceOut *hw)
    (void) hw;
 }

-static void no_enable_out(HWVoiceOut *hw, bool enable)
+static int no_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
-    NoVoiceOut *no = (NoVoiceOut *) hw;
-
-    if (enable) {
-        audio_rate_start(&no->rate);
-    }
+    (void) hw;
+    (void) cmd;
+    return 0;
 }

 static int no_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
 {
-    NoVoiceIn *no = (NoVoiceIn *) hw;
-
    audio_pcm_init_info (&hw->info, as);
    hw->samples = 1024;
-    audio_rate_start(&no->rate);
    return 0;
 }

@@ -86,22 +97,44 @@ static void no_fini_in (HWVoiceIn *hw)
    (void) hw;
 }

-static size_t no_read(HWVoiceIn *hw, void *buf, size_t size)
+static int no_run_in (HWVoiceIn *hw)
 {
    NoVoiceIn *no = (NoVoiceIn *) hw;
-    int64_t bytes = audio_rate_get_bytes(&hw->info, &no->rate, size);
+    int live = audio_pcm_hw_get_live_in (hw);
+    int dead = hw->samples - live;
+    int samples = 0;

-    audio_pcm_info_clear_buf(&hw->info, buf, bytes / hw->info.bytes_per_frame);
-    return bytes;
+    if (dead) {
+        int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        int64_t ticks = now - no->old_ticks;
+        int64_t bytes =
+            muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
+
+        no->old_ticks = now;
+        bytes = audio_MIN (bytes, INT_MAX);
+        samples = bytes >> hw->info.shift;
+        samples = audio_MIN (samples, dead);
+    }
+    return samples;
 }

-static void no_enable_in(HWVoiceIn *hw, bool enable)
+static int no_read (SWVoiceIn *sw, void *buf, int size)
 {
-    NoVoiceIn *no = (NoVoiceIn *) hw;
+    /* use custom code here instead of audio_pcm_sw_read() to avoid
+     * useless resampling/mixing */
+    int samples = size >> sw->info.shift;
+    int total = sw->hw->total_samples_captured - sw->total_hw_samples_acquired;
+    int to_clear = audio_MIN (samples, total);
+    sw->total_hw_samples_acquired += total;
+    audio_pcm_info_clear_buf (&sw->info, buf, to_clear);
+    return to_clear << sw->info.shift;
+}

-    if (enable) {
-        audio_rate_start(&no->rate);
-    }
+static int no_ctl_in (HWVoiceIn *hw, int cmd, ...)
+{
+    (void) hw;
+    (void) cmd;
+    return 0;
 }

 static void *no_audio_init(Audiodev *dev)
@@ -117,13 +150,15 @@ static void no_audio_fini (void *opaque)
 static struct audio_pcm_ops no_pcm_ops = {
    .init_out = no_init_out,
    .fini_out = no_fini_out,
+    .run_out  = no_run_out,
    .write    = no_write,
-    .enable_out = no_enable_out,
+    .ctl_out  = no_ctl_out,

    .init_in  = no_init_in,
    .fini_in  = no_fini_in,
+    .run_in   = no_run_in,
    .read     = no_read,
-    .enable_in = no_enable_in
+    .ctl_in   = no_ctl_in
 };

 static struct audio_driver no_audio_driver = {
@@ -40,15 +40,19 @@

 typedef struct OSSVoiceOut {
    HWVoiceOut hw;
+    void *pcm_buf;
    int fd;
+    int wpos;
    int nfrags;
    int fragsize;
    int mmapped;
+    int pending;
    Audiodev *dev;
 } OSSVoiceOut;

 typedef struct OSSVoiceIn {
    HWVoiceIn hw;
+    void *pcm_buf;
    int fd;
    int nfrags;
    int fragsize;
@@ -106,28 +110,33 @@ static void oss_anal_close (int *fdp)

 static void oss_helper_poll_out (void *opaque)
 {
-    AudioState *s = opaque;
-    audio_run(s, "oss_poll_out");
+    (void) opaque;
+    audio_run ("oss_poll_out");
 }

 static void oss_helper_poll_in (void *opaque)
 {
-    AudioState *s = opaque;
-    audio_run(s, "oss_poll_in");
+    (void) opaque;
+    audio_run ("oss_poll_in");
 }

 static void oss_poll_out (HWVoiceOut *hw)
 {
    OSSVoiceOut *oss = (OSSVoiceOut *) hw;

-    qemu_set_fd_handler(oss->fd, NULL, oss_helper_poll_out, hw->s);
+    qemu_set_fd_handler (oss->fd, NULL, oss_helper_poll_out, NULL);
 }

 static void oss_poll_in (HWVoiceIn *hw)
 {
    OSSVoiceIn *oss = (OSSVoiceIn *) hw;

-    qemu_set_fd_handler(oss->fd, oss_helper_poll_in, NULL, hw->s);
+    qemu_set_fd_handler (oss->fd, oss_helper_poll_in, NULL, NULL);
+}
+
+static int oss_write (SWVoiceOut *sw, void *buf, int len)
+{
+    return audio_pcm_sw_write (sw, buf, len);
 }

 static int aud_to_ossfmt (AudioFormat fmt, int endianness)
@@ -367,87 +376,97 @@ static int oss_open(int in, struct oss_params *req, audsettings *as,
    return -1;
 }

-static size_t oss_get_available_bytes(OSSVoiceOut *oss)
+static void oss_write_pending (OSSVoiceOut *oss)
 {
-    int err;
-    struct count_info cntinfo;
-    assert(oss->mmapped);
-
-    err = ioctl(oss->fd, SNDCTL_DSP_GETOPTR, &cntinfo);
-    if (err < 0) {
-        oss_logerr(errno, "SNDCTL_DSP_GETOPTR failed\n");
-        return 0;
-    }
-
-    return audio_ring_dist(cntinfo.ptr, oss->hw.pos_emul, oss->hw.size_emul);
-}
-
-static void *oss_get_buffer_out(HWVoiceOut *hw, size_t *size)
-{
-    OSSVoiceOut *oss = (OSSVoiceOut *) hw;
-    if (oss->mmapped) {
-        *size = MIN(oss_get_available_bytes(oss), hw->size_emul - hw->pos_emul);
-        return hw->buf_emul + hw->pos_emul;
-    } else {
-        return audio_generic_get_buffer_out(hw, size);
-    }
-}
-
-static size_t oss_put_buffer_out(HWVoiceOut *hw, void *buf, size_t size)
-{
-    OSSVoiceOut *oss = (OSSVoiceOut *) hw;
-    if (oss->mmapped) {
-        assert(buf == hw->buf_emul + hw->pos_emul && size < hw->size_emul);
-
-        hw->pos_emul = (hw->pos_emul + size) % hw->size_emul;
-        return size;
-    } else {
-        return audio_generic_put_buffer_out(hw, buf, size);
-    }
-}
-
-static size_t oss_write(HWVoiceOut *hw, void *buf, size_t len)
-{
-    OSSVoiceOut *oss = (OSSVoiceOut *) hw;
-    size_t pos;
+    HWVoiceOut *hw = &oss->hw;

    if (oss->mmapped) {
-        size_t total_len;
-        len = MIN(len, oss_get_available_bytes(oss));
-
-        total_len = len;
-        while (len) {
-            size_t to_copy = MIN(len, hw->size_emul - hw->pos_emul);
-            memcpy(hw->buf_emul + hw->pos_emul, buf, to_copy);
-
-            hw->pos_emul = (hw->pos_emul + to_copy) % hw->pos_emul;
-            buf += to_copy;
-            len -= to_copy;
-        }
-        return total_len;
+        return;
    }

-    pos = 0;
-    while (len) {
+    while (oss->pending) {
+        int samples_written;
        ssize_t bytes_written;
-        void *pcm = advance(buf, pos);
+        int samples_till_end = hw->samples - oss->wpos;
+        int samples_to_write = audio_MIN (oss->pending, samples_till_end);
+        int bytes_to_write = samples_to_write << hw->info.shift;
+        void *pcm = advance (oss->pcm_buf, oss->wpos << hw->info.shift);

-        bytes_written = write(oss->fd, pcm, len);
+        bytes_written = write (oss->fd, pcm, bytes_to_write);
        if (bytes_written < 0) {
            if (errno != EAGAIN) {
-                oss_logerr(errno, "failed to write %zu bytes\n",
-                           len);
+                oss_logerr (errno, "failed to write %d bytes\n",
+                            bytes_to_write);
            }
-            return pos;
-        }
-
-        pos += bytes_written;
-        if (bytes_written < len) {
            break;
        }
-        len -= bytes_written;
+
+        if (bytes_written & hw->info.align) {
+            dolog ("misaligned write asked for %d, but got %zd\n",
+                   bytes_to_write, bytes_written);
+            return;
+        }
+
+        samples_written = bytes_written >> hw->info.shift;
+        oss->pending -= samples_written;
+        oss->wpos = (oss->wpos + samples_written) % hw->samples;
+        if (bytes_written - bytes_to_write) {
+            break;
+        }
    }
-    return pos;
+}
+
+static int oss_run_out (HWVoiceOut *hw, int live)
+{
+    OSSVoiceOut *oss = (OSSVoiceOut *) hw;
+    int err, decr;
+    struct audio_buf_info abinfo;
+    struct count_info cntinfo;
+    int bufsize;
+
+    bufsize = hw->samples << hw->info.shift;
+
+    if (oss->mmapped) {
+        int bytes, pos;
+
+        err = ioctl (oss->fd, SNDCTL_DSP_GETOPTR, &cntinfo);
+        if (err < 0) {
+            oss_logerr (errno, "SNDCTL_DSP_GETOPTR failed\n");
+            return 0;
+        }
+
+        pos = hw->rpos << hw->info.shift;
+        bytes = audio_ring_dist (cntinfo.ptr, pos, bufsize);
+        decr = audio_MIN (bytes >> hw->info.shift, live);
+    }
+    else {
+        err = ioctl (oss->fd, SNDCTL_DSP_GETOSPACE, &abinfo);
+        if (err < 0) {
+            oss_logerr (errno, "SNDCTL_DSP_GETOPTR failed\n");
+            return 0;
+        }
+
+        if (abinfo.bytes > bufsize) {
+            trace_oss_invalid_available_size(abinfo.bytes, bufsize);
+            abinfo.bytes = bufsize;
+        }
+
+        if (abinfo.bytes < 0) {
+            trace_oss_invalid_available_size(abinfo.bytes, bufsize);
+            return 0;
+        }
+
+        decr = audio_MIN (abinfo.bytes >> hw->info.shift, live);
+        if (!decr) {
+            return 0;
+        }
+    }
+
+    decr = audio_pcm_hw_clip_out (hw, oss->pcm_buf, decr, oss->pending);
+    oss->pending += decr;
+    oss_write_pending (oss);
+
+    return decr;
 }

 static void oss_fini_out (HWVoiceOut *hw)
@@ -458,13 +477,18 @@ static void oss_fini_out (HWVoiceOut *hw)
    ldebug ("oss_fini\n");
    oss_anal_close (&oss->fd);

-    if (oss->mmapped && hw->buf_emul) {
-        err = munmap(hw->buf_emul, hw->size_emul);
-        if (err) {
-            oss_logerr(errno, "Failed to unmap buffer %p, size %zu\n",
-                       hw->buf_emul, hw->size_emul);
+    if (oss->pcm_buf) {
+        if (oss->mmapped) {
+            err = munmap (oss->pcm_buf, hw->samples << hw->info.shift);
+            if (err) {
+                oss_logerr (errno, "Failed to unmap buffer %p, size %d\n",
+                            oss->pcm_buf, hw->samples << hw->info.shift);
+            }
        }
-        hw->buf_emul = NULL;
+        else {
+            g_free (oss->pcm_buf);
+        }
+        oss->pcm_buf = NULL;
    }
 }

@@ -506,29 +530,28 @@ static int oss_init_out(HWVoiceOut *hw, struct audsettings *as,
    oss->nfrags = obt.nfrags;
    oss->fragsize = obt.fragsize;

-    if (obt.nfrags * obt.fragsize % hw->info.bytes_per_frame) {
+    if (obt.nfrags * obt.fragsize & hw->info.align) {
        dolog ("warning: Misaligned DAC buffer, size %d, alignment %d\n",
-               obt.nfrags * obt.fragsize, hw->info.bytes_per_frame);
+               obt.nfrags * obt.fragsize, hw->info.align + 1);
    }

-    hw->samples = (obt.nfrags * obt.fragsize) / hw->info.bytes_per_frame;
+    hw->samples = (obt.nfrags * obt.fragsize) >> hw->info.shift;

    oss->mmapped = 0;
    if (oopts->has_try_mmap && oopts->try_mmap) {
-        hw->size_emul = hw->samples * hw->info.bytes_per_frame;
-        hw->buf_emul = mmap(
+        oss->pcm_buf = mmap (
            NULL,
-            hw->size_emul,
+            hw->samples << hw->info.shift,
            PROT_READ | PROT_WRITE,
            MAP_SHARED,
            fd,
            0
            );
-        if (hw->buf_emul == MAP_FAILED) {
-            oss_logerr(errno, "Failed to map %zu bytes of DAC\n",
-                       hw->size_emul);
-            hw->buf_emul = NULL;
-        } else {
+        if (oss->pcm_buf == MAP_FAILED) {
+            oss_logerr (errno, "Failed to map %d bytes of DAC\n",
+                        hw->samples << hw->info.shift);
+        }
+        else {
            int err;
            int trig = 0;
            if (ioctl (fd, SNDCTL_DSP_SETTRIGGER, &trig) < 0) {
@@ -548,65 +571,88 @@ static int oss_init_out(HWVoiceOut *hw, struct audsettings *as,
            }

            if (!oss->mmapped) {
-                err = munmap(hw->buf_emul, hw->size_emul);
+                err = munmap (oss->pcm_buf, hw->samples << hw->info.shift);
                if (err) {
-                    oss_logerr(errno, "Failed to unmap buffer %p size %zu\n",
-                               hw->buf_emul, hw->size_emul);
+                    oss_logerr (errno, "Failed to unmap buffer %p size %d\n",
+                                oss->pcm_buf, hw->samples << hw->info.shift);
                }
-                hw->buf_emul = NULL;
            }
        }
    }

+    if (!oss->mmapped) {
+        oss->pcm_buf = audio_calloc(__func__,
+                                    hw->samples,
+                                    1 << hw->info.shift);
+        if (!oss->pcm_buf) {
+            dolog (
+                "Could not allocate DAC buffer (%d samples, each %d bytes)\n",
+                hw->samples,
+                1 << hw->info.shift
+                );
+            oss_anal_close (&fd);
+            return -1;
+        }
+    }
+
    oss->fd = fd;
    oss->dev = dev;
    return 0;
 }

-static void oss_enable_out(HWVoiceOut *hw, bool enable)
+static int oss_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
    int trig;
    OSSVoiceOut *oss = (OSSVoiceOut *) hw;
    AudiodevOssPerDirectionOptions *opdo = oss->dev->u.oss.out;

-    if (enable) {
-        bool poll_mode = opdo->try_poll;
+    switch (cmd) {
+    case VOICE_ENABLE:
+        {
+            bool poll_mode = opdo->try_poll;

-        ldebug("enabling voice\n");
-        if (poll_mode) {
-            oss_poll_out(hw);
-            poll_mode = 0;
-        }
-        hw->poll_mode = poll_mode;
+            ldebug ("enabling voice\n");
+            if (poll_mode) {
+                oss_poll_out (hw);
+                poll_mode = 0;
+            }
+            hw->poll_mode = poll_mode;

-        if (!oss->mmapped) {
-            return;
-        }
+            if (!oss->mmapped) {
+                return 0;
+            }

-        audio_pcm_info_clear_buf(&hw->info, hw->buf_emul, hw->mix_buf->size);
-        trig = PCM_ENABLE_OUTPUT;
-        if (ioctl(oss->fd, SNDCTL_DSP_SETTRIGGER, &trig) < 0) {
-            oss_logerr(errno,
-                       "SNDCTL_DSP_SETTRIGGER PCM_ENABLE_OUTPUT failed\n");
-            return;
+            audio_pcm_info_clear_buf (&hw->info, oss->pcm_buf, hw->samples);
+            trig = PCM_ENABLE_OUTPUT;
+            if (ioctl (oss->fd, SNDCTL_DSP_SETTRIGGER, &trig) < 0) {
+                oss_logerr (
+                    errno,
+                    "SNDCTL_DSP_SETTRIGGER PCM_ENABLE_OUTPUT failed\n"
+                    );
+                return -1;
+            }
        }
-    } else {
+        break;
+
+    case VOICE_DISABLE:
        if (hw->poll_mode) {
            qemu_set_fd_handler (oss->fd, NULL, NULL, NULL);
            hw->poll_mode = 0;
        }

        if (!oss->mmapped) {
-            return;
+            return 0;
        }

        ldebug ("disabling voice\n");
        trig = 0;
        if (ioctl (oss->fd, SNDCTL_DSP_SETTRIGGER, &trig) < 0) {
            oss_logerr (errno, "SNDCTL_DSP_SETTRIGGER 0 failed\n");
-            return;
+            return -1;
        }
+        break;
    }
+    return 0;
 }

 static int oss_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
@@ -644,12 +690,19 @@ static int oss_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
    oss->nfrags = obt.nfrags;
    oss->fragsize = obt.fragsize;

-    if (obt.nfrags * obt.fragsize % hw->info.bytes_per_frame) {
+    if (obt.nfrags * obt.fragsize & hw->info.align) {
        dolog ("warning: Misaligned ADC buffer, size %d, alignment %d\n",
-               obt.nfrags * obt.fragsize, hw->info.bytes_per_frame);
+               obt.nfrags * obt.fragsize, hw->info.align + 1);
    }

-    hw->samples = (obt.nfrags * obt.fragsize) / hw->info.bytes_per_frame;
+    hw->samples = (obt.nfrags * obt.fragsize) >> hw->info.shift;
+    oss->pcm_buf = audio_calloc(__func__, hw->samples, 1 << hw->info.shift);
+    if (!oss->pcm_buf) {
+        dolog ("Could not allocate ADC buffer (%d samples, each %d bytes)\n",
+               hw->samples, 1 << hw->info.shift);
+        oss_anal_close (&fd);
+        return -1;
+    }

    oss->fd = fd;
    oss->dev = dev;
@@ -661,57 +714,111 @@ static void oss_fini_in (HWVoiceIn *hw)
    OSSVoiceIn *oss = (OSSVoiceIn *) hw;

    oss_anal_close (&oss->fd);
+
+    g_free(oss->pcm_buf);
+    oss->pcm_buf = NULL;
 }

-static size_t oss_read(HWVoiceIn *hw, void *buf, size_t len)
+static int oss_run_in (HWVoiceIn *hw)
 {
    OSSVoiceIn *oss = (OSSVoiceIn *) hw;
-    size_t pos = 0;
+    int hwshift = hw->info.shift;
+    int i;
+    int live = audio_pcm_hw_get_live_in (hw);
+    int dead = hw->samples - live;
+    size_t read_samples = 0;
+    struct {
+        int add;
+        int len;
+    } bufs[2] = {
+        { .add = hw->wpos, .len = 0 },
+        { .add = 0,        .len = 0 }
+    };

-    while (len) {
+    if (!dead) {
+        return 0;
+    }
+
+    if (hw->wpos + dead > hw->samples) {
+        bufs[0].len = (hw->samples - hw->wpos) << hwshift;
+        bufs[1].len = (dead - (hw->samples - hw->wpos)) << hwshift;
+    }
+    else {
+        bufs[0].len = dead << hwshift;
+    }
+
+    for (i = 0; i < 2; ++i) {
        ssize_t nread;

-        void *dst = advance(buf, pos);
-        nread = read(oss->fd, dst, len);
+        if (bufs[i].len) {
+            void *p = advance (oss->pcm_buf, bufs[i].add << hwshift);
+            nread = read (oss->fd, p, bufs[i].len);

-        if (nread == -1) {
-            switch (errno) {
-            case EINTR:
-            case EAGAIN:
-                break;
-            default:
-                oss_logerr(errno, "Failed to read %zu bytes of audio (to %p)\n",
-                           len, dst);
+            if (nread > 0) {
+                if (nread & hw->info.align) {
+                    dolog ("warning: Misaligned read %zd (requested %d), "
+                           "alignment %d\n", nread, bufs[i].add << hwshift,
+                           hw->info.align + 1);
+                }
+                read_samples += nread >> hwshift;
+                hw->conv (hw->conv_buf + bufs[i].add, p, nread >> hwshift);
+            }
+
+            if (bufs[i].len - nread) {
+                if (nread == -1) {
+                    switch (errno) {
+                    case EINTR:
+                    case EAGAIN:
+                        break;
+                    default:
+                        oss_logerr (
+                            errno,
+                            "Failed to read %d bytes of audio (to %p)\n",
+                            bufs[i].len, p
+                            );
+                        break;
+                    }
+                }
                break;
            }
        }
-
-        pos += nread;
-        len -= nread;
    }

-    return pos;
+    hw->wpos = (hw->wpos + read_samples) % hw->samples;
+    return read_samples;
 }

-static void oss_enable_in(HWVoiceIn *hw, bool enable)
+static int oss_read (SWVoiceIn *sw, void *buf, int size)
+{
+    return audio_pcm_sw_read (sw, buf, size);
+}
+
+static int oss_ctl_in (HWVoiceIn *hw, int cmd, ...)
 {
    OSSVoiceIn *oss = (OSSVoiceIn *) hw;
    AudiodevOssPerDirectionOptions *opdo = oss->dev->u.oss.out;

-    if (enable) {
-        bool poll_mode = opdo->try_poll;
+    switch (cmd) {
+    case VOICE_ENABLE:
+        {
+            bool poll_mode = opdo->try_poll;

-        if (poll_mode) {
-            oss_poll_in(hw);
-            poll_mode = 0;
+            if (poll_mode) {
+                oss_poll_in (hw);
+                poll_mode = 0;
+            }
+            hw->poll_mode = poll_mode;
        }
-        hw->poll_mode = poll_mode;
-    } else {
+        break;
+
+    case VOICE_DISABLE:
        if (hw->poll_mode) {
            hw->poll_mode = 0;
            qemu_set_fd_handler (oss->fd, NULL, NULL, NULL);
        }
+        break;
    }
+    return 0;
 }

 static void oss_init_per_direction(AudiodevOssPerDirectionOptions *opdo)
@@ -747,15 +854,15 @@ static void oss_audio_fini (void *opaque)
 static struct audio_pcm_ops oss_pcm_ops = {
    .init_out = oss_init_out,
    .fini_out = oss_fini_out,
+    .run_out  = oss_run_out,
    .write    = oss_write,
-    .get_buffer_out = oss_get_buffer_out,
-    .put_buffer_out = oss_put_buffer_out,
-    .enable_out = oss_enable_out,
+    .ctl_out  = oss_ctl_out,

    .init_in  = oss_init_in,
    .fini_in  = oss_fini_in,
+    .run_in   = oss_run_in,
    .read     = oss_read,
-    .enable_in = oss_enable_in
+    .ctl_in   = oss_ctl_in
 };

 static struct audio_driver oss_audio_driver = {
@@ -28,7 +28,7 @@
 * Return number of samples processed.
 */
 void NAME (void *opaque, struct st_sample *ibuf, struct st_sample *obuf,
-           size_t *isamp, size_t *osamp)
+           int *isamp, int *osamp)
 {
    struct rate *rate = opaque;
    struct st_sample *istart, *iend;
@@ -41,6 +41,8 @@

 typedef struct SDLVoiceOut {
    HWVoiceOut hw;
+    int live;
+    int decr;
 } SDLVoiceOut;

 static struct SDLAudioState {
@@ -182,59 +184,67 @@ static void sdl_callback (void *opaque, Uint8 *buf, int len)
    SDLVoiceOut *sdl = opaque;
    SDLAudioState *s = &glob_sdl;
    HWVoiceOut *hw = &sdl->hw;
+    int samples = len >> hw->info.shift;
+    int to_mix, decr;

-    if (s->exit) {
+    if (s->exit || !sdl->live) {
        return;
    }

-    /* dolog ("in callback samples=%zu live=%zu\n", samples, sdl->live); */
+    /* dolog ("in callback samples=%d live=%d\n", samples, sdl->live); */

-    while (hw->pending_emul && len) {
-        size_t write_len;
-        ssize_t start = ((ssize_t) hw->pos_emul) - hw->pending_emul;
-        if (start < 0) {
-            start += hw->size_emul;
-        }
-        assert(start >= 0 && start < hw->size_emul);
+    to_mix = audio_MIN(samples, sdl->live);
+    decr = to_mix;
+    while (to_mix) {
+        int chunk = audio_MIN(to_mix, hw->samples - hw->rpos);
+        struct st_sample *src = hw->mix_buf + hw->rpos;

-        write_len = MIN(MIN(hw->pending_emul, len),
-                        hw->size_emul - start);
-
-        memcpy(buf, hw->buf_emul + start, write_len);
-        hw->pending_emul -= write_len;
-        len -= write_len;
-        buf += write_len;
+        /* dolog ("in callback to_mix %d, chunk %d\n", to_mix, chunk); */
+        hw->clip(buf, src, chunk);
+        hw->rpos = (hw->rpos + chunk) % hw->samples;
+        to_mix -= chunk;
+        buf += chunk << hw->info.shift;
    }
+    samples -= decr;
+    sdl->live -= decr;
+    sdl->decr += decr;

-    /* clear remaining buffer that we couldn't fill with data */
-    if (len) {
-        memset(buf, 0, len);
+    /* dolog ("done len=%d\n", len); */
+
+    /* SDL2 does not clear the remaining buffer for us, so do it on our own */
+    if (samples) {
+        memset(buf, 0, samples << hw->info.shift);
    }
 }

-#define SDL_WRAPPER_FUNC(name, ret_type, args_decl, args, fail, unlock) \
-    static ret_type glue(sdl_, name)args_decl                           \
-    {                                                                   \
-        ret_type ret;                                                   \
-                                                                        \
-        SDL_LockAudio();                                                \
-                                                                        \
-        ret = glue(audio_generic_, name)args;                           \
-                                                                        \
-        SDL_UnlockAudio();                                              \
-        return ret;                                                     \
+static int sdl_write_out (SWVoiceOut *sw, void *buf, int len)
+{
+    return audio_pcm_sw_write (sw, buf, len);
+}
+
+static int sdl_run_out (HWVoiceOut *hw, int live)
+{
+    int decr;
+    SDLVoiceOut *sdl = (SDLVoiceOut *) hw;
+
+    SDL_LockAudio();
+
+    if (sdl->decr > live) {
+        ldebug ("sdl->decr %d live %d sdl->live %d\n",
+                sdl->decr,
+                live,
+                sdl->live);
    }

-SDL_WRAPPER_FUNC(get_buffer_out, void *, (HWVoiceOut *hw, size_t *size),
-                 (hw, size), *size = 0, sdl_unlock)
-SDL_WRAPPER_FUNC(put_buffer_out_nowrite, size_t,
-                 (HWVoiceOut *hw, void *buf, size_t size), (hw, buf, size),
-                 /*nothing*/, sdl_unlock_and_post)
-SDL_WRAPPER_FUNC(write, size_t,
-                 (HWVoiceOut *hw, void *buf, size_t size), (hw, buf, size),
-                 /*nothing*/, sdl_unlock_and_post)
+    decr = audio_MIN (sdl->decr, live);
+    sdl->decr -= decr;

-#undef SDL_WRAPPER_FUNC
+    sdl->live = live;
+
+    SDL_UnlockAudio();
+
+    return decr;
+}

 static void sdl_fini_out (HWVoiceOut *hw)
 {
@@ -285,9 +295,20 @@ static int sdl_init_out(HWVoiceOut *hw, struct audsettings *as,
    return 0;
 }

-static void sdl_enable_out(HWVoiceOut *hw, bool enable)
+static int sdl_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
-    SDL_PauseAudio(!enable);
+    (void) hw;
+
+    switch (cmd) {
+    case VOICE_ENABLE:
+        SDL_PauseAudio (0);
+        break;
+
+    case VOICE_DISABLE:
+        SDL_PauseAudio (1);
+        break;
+    }
+    return 0;
 }

 static void *sdl_audio_init(Audiodev *dev)
@@ -320,10 +341,9 @@ static void sdl_audio_fini (void *opaque)
 static struct audio_pcm_ops sdl_pcm_ops = {
    .init_out = sdl_init_out,
    .fini_out = sdl_fini_out,
-    .write    = sdl_write,
-    .get_buffer_out = sdl_get_buffer_out,
-    .put_buffer_out = sdl_put_buffer_out_nowrite,
-    .enable_out = sdl_enable_out,
+    .run_out  = sdl_run_out,
+    .write    = sdl_write_out,
+    .ctl_out  = sdl_ctl_out,
 };

 static struct audio_driver sdl_audio_driver = {
@@ -18,6 +18,7 @@
 */

 #include "qemu/osdep.h"
+#include "hw/hw.h"
 #include "qemu/host-utils.h"
 #include "qemu/module.h"
 #include "qemu/error-report.h"
@@ -40,21 +41,27 @@
 #define LINE_IN_SAMPLES (256 * 4)
 #endif

+typedef struct SpiceRateCtl {
+    int64_t               start_ticks;
+    int64_t               bytes_sent;
+} SpiceRateCtl;
+
 typedef struct SpiceVoiceOut {
    HWVoiceOut            hw;
    SpicePlaybackInstance sin;
-    RateCtl               rate;
+    SpiceRateCtl          rate;
    int                   active;
    uint32_t              *frame;
-    uint32_t              fpos;
+    uint32_t              *fpos;
    uint32_t              fsize;
 } SpiceVoiceOut;

 typedef struct SpiceVoiceIn {
    HWVoiceIn             hw;
    SpiceRecordInstance   sin;
-    RateCtl               rate;
+    SpiceRateCtl          rate;
    int                   active;
+    uint32_t              samples[LINE_IN_SAMPLES];
 } SpiceVoiceIn;

 static const SpicePlaybackInterface playback_sif = {
@@ -84,6 +91,32 @@ static void spice_audio_fini (void *opaque)
    /* nothing */
 }

+static void rate_start (SpiceRateCtl *rate)
+{
+    memset (rate, 0, sizeof (*rate));
+    rate->start_ticks = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+}
+
+static int rate_get_samples (struct audio_pcm_info *info, SpiceRateCtl *rate)
+{
+    int64_t now;
+    int64_t ticks;
+    int64_t bytes;
+    int64_t samples;
+
+    now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    ticks = now - rate->start_ticks;
+    bytes = muldiv64(ticks, info->bytes_per_second, NANOSECONDS_PER_SECOND);
+    samples = (bytes - rate->bytes_sent) >> info->shift;
+    if (samples < 0 || samples > 65536) {
+        error_report("Resetting rate control (%" PRId64 " samples)", samples);
+        rate_start(rate);
+        samples = 0;
+    }
+    rate->bytes_sent += samples << info->shift;
+    return samples;
+}
+
 /* playback */

 static int line_out_init(HWVoiceOut *hw, struct audsettings *as,
@@ -120,78 +153,98 @@ static void line_out_fini (HWVoiceOut *hw)
    spice_server_remove_interface (&out->sin.base);
 }

-static void *line_out_get_buffer(HWVoiceOut *hw, size_t *size)
+static int line_out_run (HWVoiceOut *hw, int live)
 {
-    SpiceVoiceOut *out = container_of(hw, SpiceVoiceOut, hw);
+    SpiceVoiceOut *out = container_of (hw, SpiceVoiceOut, hw);
+    int rpos, decr;
+    int samples;

-    if (!out->frame) {
-        spice_server_playback_get_buffer(&out->sin, &out->frame, &out->fsize);
-        out->fpos = 0;
+    if (!live) {
+        return 0;
    }

-    if (out->frame) {
-        *size = audio_rate_get_bytes(
-            &hw->info, &out->rate,
-            (out->fsize - out->fpos) * hw->info.bytes_per_frame);
-    } else {
-        audio_rate_start(&out->rate);
+    decr = rate_get_samples (&hw->info, &out->rate);
+    decr = audio_MIN (live, decr);
+
+    samples = decr;
+    rpos = hw->rpos;
+    while (samples) {
+        int left_till_end_samples = hw->samples - rpos;
+        int len = audio_MIN (samples, left_till_end_samples);
+
+        if (!out->frame) {
+            spice_server_playback_get_buffer (&out->sin, &out->frame, &out->fsize);
+            out->fpos = out->frame;
+        }
+        if (out->frame) {
+            len = audio_MIN (len, out->fsize);
+            hw->clip (out->fpos, hw->mix_buf + rpos, len);
+            out->fsize -= len;
+            out->fpos  += len;
+            if (out->fsize == 0) {
+                spice_server_playback_put_samples (&out->sin, out->frame);
+                out->frame = out->fpos = NULL;
+            }
+        }
+        rpos = (rpos + len) % hw->samples;
+        samples -= len;
    }
-    return out->frame + out->fpos;
+    hw->rpos = rpos;
+    return decr;
 }

-static size_t line_out_put_buffer(HWVoiceOut *hw, void *buf, size_t size)
+static int line_out_write (SWVoiceOut *sw, void *buf, int len)
 {
-    SpiceVoiceOut *out = container_of(hw, SpiceVoiceOut, hw);
-
-    assert(buf == out->frame + out->fpos && out->fpos <= out->fsize);
-    out->fpos += size >> 2;
-
-    if (out->fpos == out->fsize) { /* buffer full */
-        spice_server_playback_put_samples(&out->sin, out->frame);
-        out->frame = NULL;
-    }
-
-    return size;
+    return audio_pcm_sw_write (sw, buf, len);
 }

-static void line_out_enable(HWVoiceOut *hw, bool enable)
+static int line_out_ctl (HWVoiceOut *hw, int cmd, ...)
 {
    SpiceVoiceOut *out = container_of (hw, SpiceVoiceOut, hw);

-    if (enable) {
+    switch (cmd) {
+    case VOICE_ENABLE:
        if (out->active) {
-            return;
+            break;
        }
        out->active = 1;
-        audio_rate_start(&out->rate);
+        rate_start (&out->rate);
        spice_server_playback_start (&out->sin);
-    } else {
+        break;
+    case VOICE_DISABLE:
        if (!out->active) {
-            return;
+            break;
        }
        out->active = 0;
        if (out->frame) {
-            memset(out->frame + out->fpos, 0, (out->fsize - out->fpos) << 2);
+            memset (out->fpos, 0, out->fsize << 2);
            spice_server_playback_put_samples (&out->sin, out->frame);
-            out->frame = NULL;
+            out->frame = out->fpos = NULL;
        }
        spice_server_playback_stop (&out->sin);
-    }
-}
-
+        break;
+    case VOICE_VOLUME:
+        {
 #if ((SPICE_INTERFACE_PLAYBACK_MAJOR >= 1) && (SPICE_INTERFACE_PLAYBACK_MINOR >= 2))
-static void line_out_volume(HWVoiceOut *hw, Volume *vol)
-{
-    SpiceVoiceOut *out = container_of(hw, SpiceVoiceOut, hw);
-    uint16_t svol[2];
+            SWVoiceOut *sw;
+            va_list ap;
+            uint16_t vol[2];

-    assert(vol->channels == 2);
-    svol[0] = vol->vol[0] * 257;
-    svol[1] = vol->vol[1] * 257;
-    spice_server_playback_set_volume(&out->sin, 2, svol);
-    spice_server_playback_set_mute(&out->sin, vol->mute);
-}
+            va_start (ap, cmd);
+            sw = va_arg (ap, SWVoiceOut *);
+            va_end (ap);
+
+            vol[0] = sw->vol.l / ((1ULL << 16) + 1);
+            vol[1] = sw->vol.r / ((1ULL << 16) + 1);
+            spice_server_playback_set_volume (&out->sin, 2, vol);
+            spice_server_playback_set_mute (&out->sin, sw->vol.mute);
 #endif
+            break;
+        }
+    }
+
+    return 0;
+}

 /* record */

@@ -228,74 +281,111 @@ static void line_in_fini (HWVoiceIn *hw)
    spice_server_remove_interface (&in->sin.base);
 }

-static size_t line_in_read(HWVoiceIn *hw, void *buf, size_t len)
+static int line_in_run (HWVoiceIn *hw)
 {
    SpiceVoiceIn *in = container_of (hw, SpiceVoiceIn, hw);
-    uint64_t to_read = audio_rate_get_bytes(&hw->info, &in->rate, len) >> 2;
-    size_t ready = spice_server_record_get_samples(&in->sin, buf, to_read);
+    int num_samples;
+    int ready;
+    int len[2];
+    uint64_t delta_samp;
+    const uint32_t *samples;

-    /* XXX: do we need this? */
-    if (ready == 0) {
-        memset(buf, 0, to_read << 2);
-        ready = to_read;
+    if (!(num_samples = hw->samples - audio_pcm_hw_get_live_in (hw))) {
+        return 0;
    }

-    return ready << 2;
+    delta_samp = rate_get_samples (&hw->info, &in->rate);
+    num_samples = audio_MIN (num_samples, delta_samp);
+
+    ready = spice_server_record_get_samples (&in->sin, in->samples, num_samples);
+    samples = in->samples;
+    if (ready == 0) {
+        static const uint32_t silence[LINE_IN_SAMPLES];
+        samples = silence;
+        ready = LINE_IN_SAMPLES;
+    }
+
+    num_samples = audio_MIN (ready, num_samples);
+
+    if (hw->wpos + num_samples > hw->samples) {
+        len[0] = hw->samples - hw->wpos;
+        len[1] = num_samples - len[0];
+    } else {
+        len[0] = num_samples;
+        len[1] = 0;
+    }
+
+    hw->conv (hw->conv_buf + hw->wpos, samples, len[0]);
+
+    if (len[1]) {
+        hw->conv (hw->conv_buf, samples + len[0], len[1]);
+    }
+
+    hw->wpos = (hw->wpos + num_samples) % hw->samples;
+
+    return num_samples;
 }

-static void line_in_enable(HWVoiceIn *hw, bool enable)
+static int line_in_read (SWVoiceIn *sw, void *buf, int size)
+{
+    return audio_pcm_sw_read (sw, buf, size);
+}
+
+static int line_in_ctl (HWVoiceIn *hw, int cmd, ...)
 {
    SpiceVoiceIn *in = container_of (hw, SpiceVoiceIn, hw);

-    if (enable) {
+    switch (cmd) {
+    case VOICE_ENABLE:
        if (in->active) {
-            return;
+            break;
        }
        in->active = 1;
-        audio_rate_start(&in->rate);
+        rate_start (&in->rate);
        spice_server_record_start (&in->sin);
-    } else {
+        break;
+    case VOICE_DISABLE:
        if (!in->active) {
-            return;
+            break;
        }
        in->active = 0;
        spice_server_record_stop (&in->sin);
-    }
-}
-
+        break;
+    case VOICE_VOLUME:
+        {
 #if ((SPICE_INTERFACE_RECORD_MAJOR >= 2) && (SPICE_INTERFACE_RECORD_MINOR >= 2))
-static void line_in_volume(HWVoiceIn *hw, Volume *vol)
-{
-    SpiceVoiceIn *in = container_of(hw, SpiceVoiceIn, hw);
-    uint16_t svol[2];
+            SWVoiceIn *sw;
+            va_list ap;
+            uint16_t vol[2];

-    assert(vol->channels == 2);
-    svol[0] = vol->vol[0] * 257;
-    svol[1] = vol->vol[1] * 257;
-    spice_server_record_set_volume(&in->sin, 2, svol);
-    spice_server_record_set_mute(&in->sin, vol->mute);
-}
+            va_start (ap, cmd);
+            sw = va_arg (ap, SWVoiceIn *);
+            va_end (ap);
+
+            vol[0] = sw->vol.l / ((1ULL << 16) + 1);
+            vol[1] = sw->vol.r / ((1ULL << 16) + 1);
+            spice_server_record_set_volume (&in->sin, 2, vol);
+            spice_server_record_set_mute (&in->sin, sw->vol.mute);
 #endif
+            break;
+        }
+    }
+
+    return 0;
+}

 static struct audio_pcm_ops audio_callbacks = {
    .init_out = line_out_init,
    .fini_out = line_out_fini,
-    .write    = audio_generic_write,
-    .get_buffer_out = line_out_get_buffer,
-    .put_buffer_out = line_out_put_buffer,
-    .enable_out = line_out_enable,
-#if (SPICE_INTERFACE_PLAYBACK_MAJOR >= 1) && \
-        (SPICE_INTERFACE_PLAYBACK_MINOR >= 2)
-    .volume_out = line_out_volume,
-#endif
+    .run_out  = line_out_run,
+    .write    = line_out_write,
+    .ctl_out  = line_out_ctl,

    .init_in  = line_in_init,
    .fini_in  = line_in_fini,
+    .run_in   = line_in_run,
    .read     = line_in_read,
-    .enable_in = line_in_enable,
-#if ((SPICE_INTERFACE_RECORD_MAJOR >= 2) && (SPICE_INTERFACE_RECORD_MINOR >= 2))
-    .volume_in = line_in_volume,
-#endif
+    .ctl_in   = line_in_ctl,
 };

 static struct audio_driver spice_audio_driver = {
@@ -308,6 +398,9 @@ static struct audio_driver spice_audio_driver = {
    .max_voices_in  = 1,
    .voice_size_out = sizeof (SpiceVoiceOut),
    .voice_size_in  = sizeof (SpiceVoiceIn),
+#if ((SPICE_INTERFACE_PLAYBACK_MAJOR >= 1) && (SPICE_INTERFACE_PLAYBACK_MINOR >= 2))
+    .ctl_caps       = VOICE_VOLUME_CAP
+#endif
 };

 void qemu_spice_audio_init (void)
@@ -35,23 +35,58 @@
 typedef struct WAVVoiceOut {
    HWVoiceOut hw;
    FILE *f;
-    RateCtl rate;
+    int64_t old_ticks;
+    void *pcm_buf;
    int total_samples;
 } WAVVoiceOut;

-static size_t wav_write_out(HWVoiceOut *hw, void *buf, size_t len)
+static int wav_run_out (HWVoiceOut *hw, int live)
 {
    WAVVoiceOut *wav = (WAVVoiceOut *) hw;
-    int64_t bytes = audio_rate_get_bytes(&hw->info, &wav->rate, len);
-    assert(bytes % hw->info.bytes_per_frame == 0);
+    int rpos, decr, samples;
+    uint8_t *dst;
+    struct st_sample *src;
+    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    int64_t ticks = now - wav->old_ticks;
+    int64_t bytes =
+        muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);

-    if (bytes && fwrite(buf, bytes, 1, wav->f) != 1) {
-        dolog("wav_write_out: fwrite of %" PRId64 " bytes failed\nReason: %s\n",
-              bytes, strerror(errno));
+    if (bytes > INT_MAX) {
+        samples = INT_MAX >> hw->info.shift;
+    }
+    else {
+        samples = bytes >> hw->info.shift;
    }

-    wav->total_samples += bytes / hw->info.bytes_per_frame;
-    return bytes;
+    wav->old_ticks = now;
+    decr = audio_MIN (live, samples);
+    samples = decr;
+    rpos = hw->rpos;
+    while (samples) {
+        int left_till_end_samples = hw->samples - rpos;
+        int convert_samples = audio_MIN (samples, left_till_end_samples);
+
+        src = hw->mix_buf + rpos;
+        dst = advance (wav->pcm_buf, rpos << hw->info.shift);
+
+        hw->clip (dst, src, convert_samples);
+        if (fwrite (dst, convert_samples << hw->info.shift, 1, wav->f) != 1) {
+            dolog ("wav_run_out: fwrite of %d bytes failed\nReaons: %s\n",
+                   convert_samples << hw->info.shift, strerror (errno));
+        }
+
+        rpos = (rpos + convert_samples) % hw->samples;
+        samples -= convert_samples;
+        wav->total_samples += convert_samples;
+    }
+
+    hw->rpos = rpos;
+    return decr;
+}
+
+static int wav_write_out (SWVoiceOut *sw, void *buf, int len)
+{
+    return audio_pcm_sw_write (sw, buf, len);
 }

 /* VICE code: Store number as little endian. */
@@ -107,6 +142,13 @@ static int wav_init_out(HWVoiceOut *hw, struct audsettings *as,
    audio_pcm_init_info (&hw->info, &wav_as);

    hw->samples = 1024;
+    wav->pcm_buf = audio_calloc(__func__, hw->samples, 1 << hw->info.shift);
+    if (!wav->pcm_buf) {
+        dolog ("Could not allocate buffer (%d bytes)\n",
+               hw->samples << hw->info.shift);
+        return -1;
+    }
+
    le_store (hdr + 22, hw->info.nchannels, 2);
    le_store (hdr + 24, hw->info.freq, 4);
    le_store (hdr + 28, hw->info.freq << (bits16 + stereo), 4);
@@ -116,6 +158,8 @@ static int wav_init_out(HWVoiceOut *hw, struct audsettings *as,
    if (!wav->f) {
        dolog ("Failed to open wave file `%s'\nReason: %s\n",
               wav_path, strerror(errno));
+        g_free (wav->pcm_buf);
+        wav->pcm_buf = NULL;
        return -1;
    }

@@ -124,8 +168,6 @@ static int wav_init_out(HWVoiceOut *hw, struct audsettings *as,
               strerror(errno));
        return -1;
    }
-
-    audio_rate_start(&wav->rate);
    return 0;
 }

@@ -134,7 +176,7 @@ static void wav_fini_out (HWVoiceOut *hw)
    WAVVoiceOut *wav = (WAVVoiceOut *) hw;
    uint8_t rlen[4];
    uint8_t dlen[4];
-    uint32_t datalen = wav->total_samples * hw->info.bytes_per_frame;
+    uint32_t datalen = wav->total_samples << hw->info.shift;
    uint32_t rifflen = datalen + 36;

    if (!wav->f) {
@@ -171,15 +213,16 @@ static void wav_fini_out (HWVoiceOut *hw)
               wav->f, strerror (errno));
    }
    wav->f = NULL;
+
+    g_free (wav->pcm_buf);
+    wav->pcm_buf = NULL;
 }

-static void wav_enable_out(HWVoiceOut *hw, bool enable)
+static int wav_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
-    WAVVoiceOut *wav = (WAVVoiceOut *) hw;
-
-    if (enable) {
-        audio_rate_start(&wav->rate);
-    }
+    (void) hw;
+    (void) cmd;
+    return 0;
 }

 static void *wav_audio_init(Audiodev *dev)
@@ -196,8 +239,9 @@ static void wav_audio_fini (void *opaque)
 static struct audio_pcm_ops wav_pcm_ops = {
    .init_out = wav_init_out,
    .fini_out = wav_fini_out,
+    .run_out  = wav_run_out,
    .write    = wav_write_out,
-    .enable_out = wav_enable_out,
+    .ctl_out  = wav_ctl_out,
 };

 static struct audio_driver wav_audio_driver = {
@@ -1,4 +1,5 @@
 #include "qemu/osdep.h"
+#include "hw/hw.h"
 #include "monitor/monitor.h"
 #include "qapi/error.h"
 #include "qemu/error-report.h"
@@ -104,8 +105,8 @@ static struct capture_ops wav_capture_ops = {
    .info = wav_capture_info
 };

-int wav_start_capture(AudioState *state, CaptureState *s, const char *path,
-                      int freq, int bits, int nchannels)
+int wav_start_capture (CaptureState *s, const char *path, int freq,
+                       int bits, int nchannels)
 {
    WAVState *wav;
    uint8_t hdr[] = {
@@ -170,7 +171,7 @@ int wav_start_capture(AudioState *state, CaptureState *s, const char *path,
        goto error_free;
    }

-    cap = AUD_add_capture(state, &as, &ops, wav);
+    cap = AUD_add_capture (&as, &ops, wav);
    if (!cap) {
        error_report("Failed to add audio capture");
        goto error_free;
@@ -1,4 +1,4 @@
-common-obj-y += rng.o rng-egd.o rng-builtin.o
+common-obj-y += rng.o rng-egd.o
 common-obj-$(CONFIG_POSIX) += rng-random.o

 common-obj-$(CONFIG_TPM) += tpm.o
@@ -23,6 +23,7 @@

 #include "qemu/osdep.h"
 #include "sysemu/cryptodev.h"
+#include "hw/boards.h"
 #include "qapi/error.h"
 #include "standard-headers/linux/virtio_crypto.h"
 #include "crypto/cipher.h"
@@ -22,6 +22,7 @@
 */

 #include "qemu/osdep.h"
+#include "hw/boards.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
@@ -23,6 +23,7 @@

 #include "qemu/osdep.h"
 #include "sysemu/cryptodev.h"
+#include "hw/boards.h"
 #include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "qemu/config-file.h"
@@ -58,6 +58,28 @@ file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp)
        return;
    }

+    /*
+     * Verify pmem file size since starting a guest with an incorrect size
+     * leads to confusing failures inside the guest.
+     */
+    if (fb->is_pmem) {
+        Error *local_err = NULL;
+        uint64_t size;
+
+        size = qemu_get_pmem_size(fb->mem_path, &local_err);
+        if (!size) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        if (backend->size > size) {
+            error_setg(errp, "size property %" PRIu64 " is larger than "
+                       "pmem file \"%s\" size %" PRIu64, backend->size,
+                       fb->mem_path, size);
+            return;
+        }
+    }
+
    backend->force_prealloc = mem_prealloc;
    name = host_memory_backend_get_name(backend);
    memory_region_init_ram_from_file(&backend->mr, OBJECT(backend),
@@ -12,7 +12,6 @@

 #include "qemu/osdep.h"
 #include "sysemu/hostmem.h"
-#include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "qapi/error.h"
 #include "qapi/qapi-builtin-visit.h"
@@ -304,7 +303,7 @@ size_t host_memory_backend_pagesize(HostMemoryBackend *memdev)
 #else
 size_t host_memory_backend_pagesize(HostMemoryBackend *memdev)
 {
-    return qemu_real_host_page_size;
+    return getpagesize();
 }
 #endif

@@ -1,77 +0,0 @@
-/*
- * QEMU Builtin Random Number Generator Backend
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- */
-
-#include "qemu/osdep.h"
-#include "sysemu/rng.h"
-#include "qemu/main-loop.h"
-#include "qemu/guest-random.h"
-
-#define RNG_BUILTIN(obj) OBJECT_CHECK(RngBuiltin, (obj), TYPE_RNG_BUILTIN)
-
-typedef struct RngBuiltin {
-    RngBackend parent;
-    QEMUBH *bh;
-} RngBuiltin;
-
-static void rng_builtin_receive_entropy_bh(void *opaque)
-{
-    RngBuiltin *s = opaque;
-
-    while (!QSIMPLEQ_EMPTY(&s->parent.requests)) {
-        RngRequest *req = QSIMPLEQ_FIRST(&s->parent.requests);
-
-        qemu_guest_getrandom_nofail(req->data, req->size);
-
-        req->receive_entropy(req->opaque, req->data, req->size);
-
-        rng_backend_finalize_request(&s->parent, req);
-    }
-}
-
-static void rng_builtin_request_entropy(RngBackend *b, RngRequest *req)
-{
-    RngBuiltin *s = RNG_BUILTIN(b);
-
-    qemu_bh_schedule(s->bh);
-}
-
-static void rng_builtin_init(Object *obj)
-{
-    RngBuiltin *s = RNG_BUILTIN(obj);
-
-    s->bh = qemu_bh_new(rng_builtin_receive_entropy_bh, s);
-}
-
-static void rng_builtin_finalize(Object *obj)
-{
-    RngBuiltin *s = RNG_BUILTIN(obj);
-
-    qemu_bh_delete(s->bh);
-}
-
-static void rng_builtin_class_init(ObjectClass *klass, void *data)
-{
-    RngBackendClass *rbc = RNG_BACKEND_CLASS(klass);
-
-    rbc->request_entropy = rng_builtin_request_entropy;
-}
-
-static const TypeInfo rng_builtin_info = {
-    .name = TYPE_RNG_BUILTIN,
-    .parent = TYPE_RNG_BACKEND,
-    .instance_size = sizeof(RngBuiltin),
-    .instance_init = rng_builtin_init,
-    .instance_finalize = rng_builtin_finalize,
-    .class_init = rng_builtin_class_init,
-};
-
-static void register_types(void)
-{
-    type_register_static(&rng_builtin_info);
-}
-
-type_init(register_types);
@@ -12,6 +12,7 @@


 #include "qemu/osdep.h"
+#include "hw/qdev.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
@@ -46,7 +47,7 @@ vhost_user_backend_dev_init(VhostUserBackend *b, VirtIODevice *vdev,

    b->vdev = vdev;
    b->dev.nvqs = nvqs;
-    b->dev.vqs = g_new0(struct vhost_virtqueue, nvqs);
+    b->dev.vqs = g_new(struct vhost_virtqueue, nvqs);

    ret = vhost_dev_init(&b->dev, &b->vhost_user, VHOST_BACKEND_TYPE_USER, 0);
    if (ret < 0) {
@@ -26,6 +26,7 @@

 #include "qemu/osdep.h"
 #include "qemu/atomic.h"
+#include "exec/cpu-common.h"
 #include "sysemu/kvm.h"
 #include "sysemu/balloon.h"
 #include "trace-root.h"
@@ -30,7 +30,6 @@
 #include "block/qdict.h"
 #include "qemu/error-report.h"
 #include "module_block.h"
-#include "qemu/main-loop.h"
 #include "qemu/module.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
@@ -106,7 +105,7 @@ size_t bdrv_opt_mem_align(BlockDriverState *bs)
 {
    if (!bs || !bs->drv) {
        /* page size or 4k (hdd sector size) should be on the safe side */
-        return MAX(4096, qemu_real_host_page_size);
+        return MAX(4096, getpagesize());
    }

    return bs->bl.opt_mem_alignment;
@@ -116,7 +115,7 @@ size_t bdrv_min_mem_align(BlockDriverState *bs)
 {
    if (!bs || !bs->drv) {
        /* page size or 4k (hdd sector size) should be on the safe side */
-        return MAX(4096, qemu_real_host_page_size);
+        return MAX(4096, getpagesize());
    }

    return bs->bl.min_mem_alignment;
@@ -1719,7 +1718,7 @@ typedef struct BlockReopenQueueEntry {
     bool prepared;
     bool perms_checked;
     BDRVReopenState state;
-     QTAILQ_ENTRY(BlockReopenQueueEntry) entry;
+     QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
 } BlockReopenQueueEntry;

 /*
@@ -1732,7 +1731,7 @@ static int bdrv_reopen_get_flags(BlockReopenQueue *q, BlockDriverState *bs)
    BlockReopenQueueEntry *entry;

    if (q != NULL) {
-        QTAILQ_FOREACH(entry, q, entry) {
+        QSIMPLEQ_FOREACH(entry, q, entry) {
            if (entry->state.bs == bs) {
                return entry->state.flags;
            }
@@ -2169,8 +2168,16 @@ void bdrv_filter_default_perms(BlockDriverState *bs, BdrvChild *c,
                               uint64_t perm, uint64_t shared,
                               uint64_t *nperm, uint64_t *nshared)
 {
-    *nperm = perm & DEFAULT_PERM_PASSTHROUGH;
-    *nshared = (shared & DEFAULT_PERM_PASSTHROUGH) | DEFAULT_PERM_UNCHANGED;
+    if (c == NULL) {
+        *nperm = perm & DEFAULT_PERM_PASSTHROUGH;
+        *nshared = (shared & DEFAULT_PERM_PASSTHROUGH) | DEFAULT_PERM_UNCHANGED;
+        return;
+    }
+
+    *nperm = (perm & DEFAULT_PERM_PASSTHROUGH) |
+             (c->perm & DEFAULT_PERM_UNCHANGED);
+    *nshared = (shared & DEFAULT_PERM_PASSTHROUGH) |
+               (c->shared_perm & DEFAULT_PERM_UNCHANGED);
 }

 void bdrv_format_default_perms(BlockDriverState *bs, BdrvChild *c,
@@ -2231,27 +2238,13 @@ static void bdrv_replace_child_noperm(BdrvChild *child,
                                      BlockDriverState *new_bs)
 {
    BlockDriverState *old_bs = child->bs;
-    int new_bs_quiesce_counter;
-    int drain_saldo;
+    int i;

    assert(!child->frozen);

    if (old_bs && new_bs) {
        assert(bdrv_get_aio_context(old_bs) == bdrv_get_aio_context(new_bs));
    }
-
-    new_bs_quiesce_counter = (new_bs ? new_bs->quiesce_counter : 0);
-    drain_saldo = new_bs_quiesce_counter - child->parent_quiesce_counter;
-
-    /*
-     * If the new child node is drained but the old one was not, flush
-     * all outstanding requests to the old child node.
-     */
-    while (drain_saldo > 0 && child->role->drained_begin) {
-        bdrv_parent_drained_begin_single(child, true);
-        drain_saldo--;
-    }
-
    if (old_bs) {
        /* Detach first so that the recursive drain sections coming from @child
         * are already gone and we only end the drain sections that came from
@@ -2259,22 +2252,28 @@ static void bdrv_replace_child_noperm(BdrvChild *child,
        if (child->role->detach) {
            child->role->detach(child);
        }
+        while (child->parent_quiesce_counter) {
+            bdrv_parent_drained_end_single(child);
+        }
        QLIST_REMOVE(child, next_parent);
+    } else {
+        assert(child->parent_quiesce_counter == 0);
    }

    child->bs = new_bs;

    if (new_bs) {
        QLIST_INSERT_HEAD(&new_bs->parents, child, next_parent);
-
-        /*
-         * Detaching the old node may have led to the new node's
-         * quiesce_counter having been decreased.  Not a problem, we
-         * just need to recognize this here and then invoke
-         * drained_end appropriately more often.
-         */
-        assert(new_bs->quiesce_counter <= new_bs_quiesce_counter);
-        drain_saldo += new_bs->quiesce_counter - new_bs_quiesce_counter;
+        if (new_bs->quiesce_counter) {
+            int num = new_bs->quiesce_counter;
+            if (child->role->parent_is_bds) {
+                num -= bdrv_drain_all_count;
+            }
+            assert(num >= 0);
+            for (i = 0; i < num; i++) {
+                bdrv_parent_drained_begin_single(child, true);
+            }
+        }

        /* Attach only after starting new drained sections, so that recursive
         * drain sections coming from @child don't get an extra .drained_begin
@@ -2283,15 +2282,6 @@ static void bdrv_replace_child_noperm(BdrvChild *child,
            child->role->attach(child);
        }
    }
-
-    /*
-     * If the old child node was drained but the new one is not, allow
-     * requests to come in only after the new node has been attached.
-     */
-    while (drain_saldo < 0 && child->role->drained_end) {
-        bdrv_parent_drained_end_single(child);
-        drain_saldo++;
-    }
 }

 /*
@@ -3249,7 +3239,7 @@ static bool bdrv_recurse_has_child(BlockDriverState *bs,
 * Adds a BlockDriverState to a simple queue for an atomic, transactional
 * reopen of multiple devices.
 *
- * bs_queue can either be an existing BlockReopenQueue that has had QTAILQ_INIT
+ * bs_queue can either be an existing BlockReopenQueue that has had QSIMPLE_INIT
 * already performed, or alternatively may be NULL a new BlockReopenQueue will
 * be created and initialized. This newly created BlockReopenQueue should be
 * passed back in for subsequent calls that are intended to be of the same
@@ -3290,7 +3280,7 @@ static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,

    if (bs_queue == NULL) {
        bs_queue = g_new0(BlockReopenQueue, 1);
-        QTAILQ_INIT(bs_queue);
+        QSIMPLEQ_INIT(bs_queue);
    }

    if (!options) {
@@ -3298,7 +3288,7 @@ static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
    }

    /* Check if this BlockDriverState is already in the queue */
-    QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
        if (bs == bs_entry->state.bs) {
            break;
        }
@@ -3354,7 +3344,7 @@ static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,

    if (!bs_entry) {
        bs_entry = g_new0(BlockReopenQueueEntry, 1);
-        QTAILQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+        QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
    } else {
        qobject_unref(bs_entry->state.options);
        qobject_unref(bs_entry->state.explicit_options);
@@ -3455,7 +3445,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)

    assert(bs_queue != NULL);

-    QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
        assert(bs_entry->state.bs->quiesce_counter > 0);
        if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, errp)) {
            goto cleanup;
@@ -3463,7 +3453,7 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)
        bs_entry->prepared = true;
    }

-    QTAILQ_FOREACH(bs_entry, bs_queue, entry) {
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
        BDRVReopenState *state = &bs_entry->state;
        ret = bdrv_check_perm(state->bs, bs_queue, state->perm,
                              state->shared_perm, NULL, NULL, errp);
@@ -3486,22 +3476,16 @@ int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)
        bs_entry->perms_checked = true;
    }

-    /*
-     * If we reach this point, we have success and just need to apply the
-     * changes.
-     *
-     * Reverse order is used to comfort qcow2 driver: on commit it need to write
-     * IN_USE flag to the image, to mark bitmaps in the image as invalid. But
-     * children are usually goes after parents in reopen-queue, so go from last
-     * to first element.
+    /* If we reach this point, we have success and just need to apply the
+     * changes
     */
-    QTAILQ_FOREACH_REVERSE(bs_entry, bs_queue, entry) {
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
        bdrv_reopen_commit(&bs_entry->state);
    }

    ret = 0;
 cleanup_perm:
-    QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+    QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
        BDRVReopenState *state = &bs_entry->state;

        if (!bs_entry->perms_checked) {
@@ -3518,7 +3502,7 @@ cleanup_perm:
        }
    }
 cleanup:
-    QTAILQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+    QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
        if (ret) {
            if (bs_entry->prepared) {
                bdrv_reopen_abort(&bs_entry->state);
@@ -3558,7 +3542,7 @@ static BlockReopenQueueEntry *find_parent_in_reopen_queue(BlockReopenQueue *q,
 {
    BlockReopenQueueEntry *entry;

-    QTAILQ_FOREACH(entry, q, entry) {
+    QSIMPLEQ_FOREACH(entry, q, entry) {
        BlockDriverState *bs = entry->state.bs;
        BdrvChild *child;

@@ -3935,12 +3919,16 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
    BlockDriver *drv;
    BlockDriverState *bs;
    BdrvChild *child;
+    bool old_can_write, new_can_write;

    assert(reopen_state != NULL);
    bs = reopen_state->bs;
    drv = bs->drv;
    assert(drv != NULL);

+    old_can_write =
+        !bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
+
    /* If there are any driver level actions to take */
    if (drv->bdrv_reopen_commit) {
        drv->bdrv_reopen_commit(reopen_state);
@@ -3984,6 +3972,21 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
    }

    bdrv_refresh_limits(bs, NULL);
+
+    new_can_write =
+        !bdrv_is_read_only(bs) && !(bdrv_get_flags(bs) & BDRV_O_INACTIVE);
+    if (!old_can_write && new_can_write && drv->bdrv_reopen_bitmaps_rw) {
+        Error *local_err = NULL;
+        if (drv->bdrv_reopen_bitmaps_rw(bs, &local_err) < 0) {
+            /* This is not fatal, bitmaps just left read-only, so all following
+             * writes will fail. User can remove read-only bitmaps to unblock
+             * writes.
+             */
+            error_reportf_err(local_err,
+                              "%s: Failed to make dirty bitmaps writable: ",
+                              bdrv_get_node_name(bs));
+        }
+    }
 }

 /*
@@ -4152,6 +4155,7 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
 {
    BdrvChild *c, *next;
    GSList *list = NULL, *p;
+    uint64_t old_perm, old_shared;
    uint64_t perm = 0, shared = BLK_PERM_ALL;
    int ret;

@@ -4197,8 +4201,8 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
        bdrv_unref(from);
    }

-    bdrv_get_cumulative_perm(to, &perm, &shared);
-    bdrv_set_perm(to, perm, shared);
+    bdrv_get_cumulative_perm(to, &old_perm, &old_shared);
+    bdrv_set_perm(to, old_perm | perm, old_shared | shared);

 out:
    g_slist_free(list);
@@ -4495,7 +4499,6 @@ int bdrv_drop_intermediate(BlockDriverState *top, BlockDriverState *base,
    int ret = -EIO;

    bdrv_ref(top);
-    bdrv_subtree_drained_begin(top);

    if (!top->drv || !base->drv) {
        goto exit;
@@ -4567,7 +4570,6 @@ int bdrv_drop_intermediate(BlockDriverState *top, BlockDriverState *base,

    ret = 0;
 exit:
-    bdrv_subtree_drained_end(top);
    bdrv_unref(top);
    return ret;
 }
@@ -5064,27 +5066,6 @@ int bdrv_has_zero_init(BlockDriverState *bs)
    return 0;
 }

-int bdrv_has_zero_init_truncate(BlockDriverState *bs)
-{
-    if (!bs->drv) {
-        return 0;
-    }
-
-    if (bs->backing) {
-        /* Depends on the backing image length, but better safe than sorry */
-        return 0;
-    }
-    if (bs->drv->bdrv_has_zero_init_truncate) {
-        return bs->drv->bdrv_has_zero_init_truncate(bs);
-    }
-    if (bs->file && bs->drv->is_filter) {
-        return bdrv_has_zero_init_truncate(bs->file->bs);
-    }
-
-    /* safe default */
-    return 0;
-}
-
 bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs)
 {
    BlockDriverInfo bdi;
@@ -5142,15 +5123,6 @@ ImageInfoSpecific *bdrv_get_specific_info(BlockDriverState *bs,
    return NULL;
 }

-BlockStatsSpecific *bdrv_get_specific_stats(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv || !drv->bdrv_get_specific_stats) {
-        return NULL;
-    }
-    return drv->bdrv_get_specific_stats(bs);
-}
-
 void bdrv_debug_event(BlockDriverState *bs, BlkdebugEvent event)
 {
    if (!bs || !bs->drv || !bs->drv->bdrv_debug_event) {
@@ -5160,35 +5132,14 @@ void bdrv_debug_event(BlockDriverState *bs, BlkdebugEvent event)
    bs->drv->bdrv_debug_event(bs, event);
 }

-static BlockDriverState *bdrv_find_debug_node(BlockDriverState *bs)
-{
-    while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
-        if (bs->file) {
-            bs = bs->file->bs;
-            continue;
-        }
-
-        if (bs->drv->is_filter && bs->backing) {
-            bs = bs->backing->bs;
-            continue;
-        }
-
-        break;
-    }
-
-    if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
-        assert(bs->drv->bdrv_debug_remove_breakpoint);
-        return bs;
-    }
-
-    return NULL;
-}
-
 int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
                          const char *tag)
 {
-    bs = bdrv_find_debug_node(bs);
-    if (bs) {
+    while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
+        bs = bs->file ? bs->file->bs : NULL;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
        return bs->drv->bdrv_debug_breakpoint(bs, event, tag);
    }

@@ -5197,8 +5148,11 @@ int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,

 int bdrv_debug_remove_breakpoint(BlockDriverState *bs, const char *tag)
 {
-    bs = bdrv_find_debug_node(bs);
-    if (bs) {
+    while (bs && bs->drv && !bs->drv->bdrv_debug_remove_breakpoint) {
+        bs = bs->file ? bs->file->bs : NULL;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_remove_breakpoint) {
        return bs->drv->bdrv_debug_remove_breakpoint(bs, tag);
    }

@@ -5377,8 +5331,10 @@ static void coroutine_fn bdrv_co_invalidate_cache(BlockDriverState *bs,
        }
    }

-    FOR_EACH_DIRTY_BITMAP(bs, bm) {
-        bdrv_dirty_bitmap_skip_store(bm, false);
+    for (bm = bdrv_dirty_bitmap_next(bs, NULL); bm;
+         bm = bdrv_dirty_bitmap_next(bs, bm))
+    {
+        bdrv_dirty_bitmap_set_migration(bm, false);
    }

    ret = refresh_total_sectors(bs, bs->total_sectors);
@@ -6567,3 +6523,25 @@ void bdrv_del_child(BlockDriverState *parent_bs, BdrvChild *child, Error **errp)

    parent_bs->drv->bdrv_del_child(parent_bs, child, errp);
 }
+
+bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
+                                     uint32_t granularity, Error **errp)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (!drv) {
+        error_setg_errno(errp, ENOMEDIUM,
+                         "Can't store persistent bitmaps to %s",
+                         bdrv_get_device_or_node_name(bs));
+        return false;
+    }
+
+    if (!drv->bdrv_can_store_new_dirty_bitmap) {
+        error_setg_errno(errp, ENOTSUP,
+                         "Can't store persistent bitmaps to %s",
+                         bdrv_get_device_or_node_name(bs));
+        return false;
+    }
+
+    return drv->bdrv_can_store_new_dirty_bitmap(bs, name, granularity, errp);
+}
@@ -37,13 +37,9 @@ block-obj-y += write-threshold.o
 block-obj-y += backup.o
 block-obj-$(CONFIG_REPLICATION) += replication.o
 block-obj-y += throttle.o copy-on-read.o
-block-obj-y += block-copy.o

 block-obj-y += crypto.o

-block-obj-y += aio_task.o
-block-obj-y += backup-top.o
-
 common-obj-y += stream.o

 nfs.o-libs         := $(LIBNFS_LIBS)
@@ -195,10 +195,6 @@ static void block_account_one_io(BlockAcctStats *stats, BlockAcctCookie *cookie,

    assert(cookie->type < BLOCK_MAX_IOTYPE);

-    if (cookie->type == BLOCK_ACCT_NONE) {
-        return;
-    }
-
    qemu_mutex_lock(&stats->lock);

    if (failed) {
@@ -221,8 +217,6 @@ static void block_account_one_io(BlockAcctStats *stats, BlockAcctCookie *cookie,
    }

    qemu_mutex_unlock(&stats->lock);
-
-    cookie->type = BLOCK_ACCT_NONE;
 }

 void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie)
@@ -1,124 +0,0 @@
-/*
- * Aio tasks loops
- *
- * Copyright (c) 2019 Virtuozzo International GmbH.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu/osdep.h"
-#include "block/aio.h"
-#include "block/aio_task.h"
-
-struct AioTaskPool {
-    Coroutine *main_co;
-    int status;
-    int max_busy_tasks;
-    int busy_tasks;
-    bool waiting;
-};
-
-static void coroutine_fn aio_task_co(void *opaque)
-{
-    AioTask *task = opaque;
-    AioTaskPool *pool = task->pool;
-
-    assert(pool->busy_tasks < pool->max_busy_tasks);
-    pool->busy_tasks++;
-
-    task->ret = task->func(task);
-
-    pool->busy_tasks--;
-
-    if (task->ret < 0 && pool->status == 0) {
-        pool->status = task->ret;
-    }
-
-    g_free(task);
-
-    if (pool->waiting) {
-        pool->waiting = false;
-        aio_co_wake(pool->main_co);
-    }
-}
-
-void coroutine_fn aio_task_pool_wait_one(AioTaskPool *pool)
-{
-    assert(pool->busy_tasks > 0);
-    assert(qemu_coroutine_self() == pool->main_co);
-
-    pool->waiting = true;
-    qemu_coroutine_yield();
-
-    assert(!pool->waiting);
-    assert(pool->busy_tasks < pool->max_busy_tasks);
-}
-
-void coroutine_fn aio_task_pool_wait_slot(AioTaskPool *pool)
-{
-    if (pool->busy_tasks < pool->max_busy_tasks) {
-        return;
-    }
-
-    aio_task_pool_wait_one(pool);
-}
-
-void coroutine_fn aio_task_pool_wait_all(AioTaskPool *pool)
-{
-    while (pool->busy_tasks > 0) {
-        aio_task_pool_wait_one(pool);
-    }
-}
-
-void coroutine_fn aio_task_pool_start_task(AioTaskPool *pool, AioTask *task)
-{
-    aio_task_pool_wait_slot(pool);
-
-    task->pool = pool;
-    qemu_coroutine_enter(qemu_coroutine_create(aio_task_co, task));
-}
-
-AioTaskPool *coroutine_fn aio_task_pool_new(int max_busy_tasks)
-{
-    AioTaskPool *pool = g_new0(AioTaskPool, 1);
-
-    pool->main_co = qemu_coroutine_self();
-    pool->max_busy_tasks = max_busy_tasks;
-
-    return pool;
-}
-
-void aio_task_pool_free(AioTaskPool *pool)
-{
-    g_free(pool);
-}
-
-int aio_task_pool_status(AioTaskPool *pool)
-{
-    if (!pool) {
-        return 0; /* Sugar for lazy allocation of aio pool */
-    }
-
-    return pool->status;
-}
-
-bool aio_task_pool_empty(AioTaskPool *pool)
-{
-    return pool->busy_tasks == 0;
-}
@@ -1,276 +0,0 @@
-/*
- * backup-top filter driver
- *
- * The driver performs Copy-Before-Write (CBW) operation: it is injected above
- * some node, and before each write it copies _old_ data to the target node.
- *
- * Copyright (c) 2018-2019 Virtuozzo International GmbH.
- *
- * Author:
- *  Sementsov-Ogievskiy Vladimir <vsementsov@virtuozzo.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "qemu/osdep.h"
-
-#include "sysemu/block-backend.h"
-#include "qemu/cutils.h"
-#include "qapi/error.h"
-#include "block/block_int.h"
-#include "block/qdict.h"
-#include "block/block-copy.h"
-
-#include "block/backup-top.h"
-
-typedef struct BDRVBackupTopState {
-    BlockCopyState *bcs;
-    BdrvChild *target;
-    bool active;
-} BDRVBackupTopState;
-
-static coroutine_fn int backup_top_co_preadv(
-        BlockDriverState *bs, uint64_t offset, uint64_t bytes,
-        QEMUIOVector *qiov, int flags)
-{
-    return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
-}
-
-static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset,
-                                       uint64_t bytes)
-{
-    BDRVBackupTopState *s = bs->opaque;
-    uint64_t end = QEMU_ALIGN_UP(offset + bytes, s->bcs->cluster_size);
-    uint64_t off = QEMU_ALIGN_DOWN(offset, s->bcs->cluster_size);
-
-    return block_copy(s->bcs, off, end - off, NULL);
-}
-
-static int coroutine_fn backup_top_co_pdiscard(BlockDriverState *bs,
-                                               int64_t offset, int bytes)
-{
-    int ret = backup_top_cbw(bs, offset, bytes);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return bdrv_co_pdiscard(bs->backing, offset, bytes);
-}
-
-static int coroutine_fn backup_top_co_pwrite_zeroes(BlockDriverState *bs,
-        int64_t offset, int bytes, BdrvRequestFlags flags)
-{
-    int ret = backup_top_cbw(bs, offset, bytes);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
-}
-
-static coroutine_fn int backup_top_co_pwritev(BlockDriverState *bs,
-                                              uint64_t offset,
-                                              uint64_t bytes,
-                                              QEMUIOVector *qiov, int flags)
-{
-    if (!(flags & BDRV_REQ_WRITE_UNCHANGED)) {
-        int ret = backup_top_cbw(bs, offset, bytes);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    return bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
-}
-
-static int coroutine_fn backup_top_co_flush(BlockDriverState *bs)
-{
-    if (!bs->backing) {
-        return 0;
-    }
-
-    return bdrv_co_flush(bs->backing->bs);
-}
-
-static void backup_top_refresh_filename(BlockDriverState *bs)
-{
-    if (bs->backing == NULL) {
-        /*
-         * we can be here after failed bdrv_attach_child in
-         * bdrv_set_backing_hd
-         */
-        return;
-    }
-    pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
-            bs->backing->bs->filename);
-}
-
-static void backup_top_child_perm(BlockDriverState *bs, BdrvChild *c,
-                                  const BdrvChildRole *role,
-                                  BlockReopenQueue *reopen_queue,
-                                  uint64_t perm, uint64_t shared,
-                                  uint64_t *nperm, uint64_t *nshared)
-{
-    BDRVBackupTopState *s = bs->opaque;
-
-    if (!s->active) {
-        /*
-         * The filter node may be in process of bdrv_append(), which firstly do
-         * bdrv_set_backing_hd() and then bdrv_replace_node(). This means that
-         * we can't unshare BLK_PERM_WRITE during bdrv_append() operation. So,
-         * let's require nothing during bdrv_append() and refresh permissions
-         * after it (see bdrv_backup_top_append()).
-         */
-        *nperm = 0;
-        *nshared = BLK_PERM_ALL;
-        return;
-    }
-
-    if (role == &child_file) {
-        /*
-         * Target child
-         *
-         * Share write to target (child_file), to not interfere
-         * with guest writes to its disk which may be in target backing chain.
-         */
-        *nshared = BLK_PERM_ALL;
-        *nperm = BLK_PERM_WRITE;
-    } else {
-        /* Source child */
-        bdrv_filter_default_perms(bs, c, role, reopen_queue, perm, shared,
-                                  nperm, nshared);
-
-        if (perm & BLK_PERM_WRITE) {
-            *nperm = *nperm | BLK_PERM_CONSISTENT_READ;
-        }
-        *nshared &= ~BLK_PERM_WRITE;
-    }
-}
-
-BlockDriver bdrv_backup_top_filter = {
-    .format_name = "backup-top",
-    .instance_size = sizeof(BDRVBackupTopState),
-
-    .bdrv_co_preadv             = backup_top_co_preadv,
-    .bdrv_co_pwritev            = backup_top_co_pwritev,
-    .bdrv_co_pwrite_zeroes      = backup_top_co_pwrite_zeroes,
-    .bdrv_co_pdiscard           = backup_top_co_pdiscard,
-    .bdrv_co_flush              = backup_top_co_flush,
-
-    .bdrv_co_block_status       = bdrv_co_block_status_from_backing,
-
-    .bdrv_refresh_filename      = backup_top_refresh_filename,
-
-    .bdrv_child_perm            = backup_top_child_perm,
-
-    .is_filter = true,
-};
-
-BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,
-                                         BlockDriverState *target,
-                                         const char *filter_node_name,
-                                         uint64_t cluster_size,
-                                         BdrvRequestFlags write_flags,
-                                         BlockCopyState **bcs,
-                                         Error **errp)
-{
-    Error *local_err = NULL;
-    BDRVBackupTopState *state;
-    BlockDriverState *top = bdrv_new_open_driver(&bdrv_backup_top_filter,
-                                                 filter_node_name,
-                                                 BDRV_O_RDWR, errp);
-
-    if (!top) {
-        return NULL;
-    }
-
-    top->total_sectors = source->total_sectors;
-    top->opaque = state = g_new0(BDRVBackupTopState, 1);
-
-    bdrv_ref(target);
-    state->target = bdrv_attach_child(top, target, "target", &child_file, errp);
-    if (!state->target) {
-        bdrv_unref(target);
-        bdrv_unref(top);
-        return NULL;
-    }
-
-    bdrv_drained_begin(source);
-
-    bdrv_ref(top);
-    bdrv_append(top, source, &local_err);
-    if (local_err) {
-        error_prepend(&local_err, "Cannot append backup-top filter: ");
-        goto append_failed;
-    }
-
-    /*
-     * bdrv_append() finished successfully, now we can require permissions
-     * we want.
-     */
-    state->active = true;
-    bdrv_child_refresh_perms(top, top->backing, &local_err);
-    if (local_err) {
-        error_prepend(&local_err,
-                      "Cannot set permissions for backup-top filter: ");
-        goto failed_after_append;
-    }
-
-    state->bcs = block_copy_state_new(top->backing, state->target,
-                                      cluster_size, write_flags, &local_err);
-    if (local_err) {
-        error_prepend(&local_err, "Cannot create block-copy-state: ");
-        goto failed_after_append;
-    }
-    *bcs = state->bcs;
-
-    bdrv_drained_end(source);
-
-    return top;
-
-failed_after_append:
-    state->active = false;
-    bdrv_backup_top_drop(top);
-
-append_failed:
-    bdrv_drained_end(source);
-    bdrv_unref_child(top, state->target);
-    bdrv_unref(top);
-    error_propagate(errp, local_err);
-
-    return NULL;
-}
-
-void bdrv_backup_top_drop(BlockDriverState *bs)
-{
-    BDRVBackupTopState *s = bs->opaque;
-    AioContext *aio_context = bdrv_get_aio_context(bs);
-
-    block_copy_state_free(s->bcs);
-
-    aio_context_acquire(aio_context);
-
-    bdrv_drained_begin(bs);
-
-    s->active = false;
-    bdrv_child_refresh_perms(bs, bs->backing, &error_abort);
-    bdrv_replace_node(bs, backing_bs(bs), &error_abort);
-    bdrv_set_backing_hd(bs, NULL, &error_abort);
-
-    bdrv_drained_end(bs);
-
-    bdrv_unref(bs);
-
-    aio_context_release(aio_context);
-}
@@ -1,41 +0,0 @@
-/*
- * backup-top filter driver
- *
- * The driver performs Copy-Before-Write (CBW) operation: it is injected above
- * some node, and before each write it copies _old_ data to the target node.
- *
- * Copyright (c) 2018-2019 Virtuozzo International GmbH.
- *
- * Author:
- *  Sementsov-Ogievskiy Vladimir <vsementsov@virtuozzo.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef BACKUP_TOP_H
-#define BACKUP_TOP_H
-
-#include "block/block_int.h"
-#include "block/block-copy.h"
-
-BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,
-                                         BlockDriverState *target,
-                                         const char *filter_node_name,
-                                         uint64_t cluster_size,
-                                         BdrvRequestFlags write_flags,
-                                         BlockCopyState **bcs,
-                                         Error **errp);
-void bdrv_backup_top_drop(BlockDriverState *bs);
-
-#endif /* BACKUP_TOP_H */
@@ -2,7 +2,6 @@
 * QEMU backup
 *
 * Copyright (C) 2013 Proxmox Server Solutions
- * Copyright (c) 2019 Virtuozzo International GmbH.
 *
 * Authors:
 *  Dietmar Maurer (dietmar@proxmox.com)
@@ -19,7 +18,6 @@
 #include "block/block_int.h"
 #include "block/blockjob_int.h"
 #include "block/block_backup.h"
-#include "block/block-copy.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
@@ -28,91 +26,262 @@
 #include "qemu/bitmap.h"
 #include "qemu/error-report.h"

-#include "block/backup-top.h"
-
 #define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16)

+typedef struct CowRequest {
+    int64_t start_byte;
+    int64_t end_byte;
+    QLIST_ENTRY(CowRequest) list;
+    CoQueue wait_queue; /* coroutines blocked on this request */
+} CowRequest;
+
 typedef struct BackupBlockJob {
    BlockJob common;
-    BlockDriverState *backup_top;
-    BlockDriverState *source_bs;
-
+    BlockBackend *target;
+    /* bitmap for sync=incremental */
    BdrvDirtyBitmap *sync_bitmap;
-
    MirrorSyncMode sync_mode;
-    BitmapSyncMode bitmap_mode;
    BlockdevOnError on_source_error;
    BlockdevOnError on_target_error;
+    CoRwlock flush_rwlock;
    uint64_t len;
    uint64_t bytes_read;
    int64_t cluster_size;
+    bool compress;
+    NotifierWithReturn before_write;
+    QLIST_HEAD(, CowRequest) inflight_reqs;

-    BlockCopyState *bcs;
+    HBitmap *copy_bitmap;
+    bool use_copy_range;
+    int64_t copy_range_size;
+
+    bool serialize_target_writes;
 } BackupBlockJob;

 static const BlockJobDriver backup_job_driver;

-static void backup_progress_bytes_callback(int64_t bytes, void *opaque)
+/* See if in-flight requests overlap and wait for them to complete */
+static void coroutine_fn wait_for_overlapping_requests(BackupBlockJob *job,
+                                                       int64_t start,
+                                                       int64_t end)
 {
-    BackupBlockJob *s = opaque;
+    CowRequest *req;
+    bool retry;

-    s->bytes_read += bytes;
-    job_progress_update(&s->common.job, bytes);
+    do {
+        retry = false;
+        QLIST_FOREACH(req, &job->inflight_reqs, list) {
+            if (end > req->start_byte && start < req->end_byte) {
+                qemu_co_queue_wait(&req->wait_queue, NULL);
+                retry = true;
+                break;
+            }
+        }
+    } while (retry);
 }

-static void backup_progress_reset_callback(void *opaque)
+/* Keep track of an in-flight request */
+static void cow_request_begin(CowRequest *req, BackupBlockJob *job,
+                              int64_t start, int64_t end)
 {
-    BackupBlockJob *s = opaque;
-    uint64_t estimate = bdrv_get_dirty_count(s->bcs->copy_bitmap);
+    req->start_byte = start;
+    req->end_byte = end;
+    qemu_co_queue_init(&req->wait_queue);
+    QLIST_INSERT_HEAD(&job->inflight_reqs, req, list);
+}

-    job_progress_set_remaining(&s->common.job, estimate);
+/* Forget about a completed request */
+static void cow_request_end(CowRequest *req)
+{
+    QLIST_REMOVE(req, list);
+    qemu_co_queue_restart_all(&req->wait_queue);
+}
+
+/* Copy range to target with a bounce buffer and return the bytes copied. If
+ * error occurred, return a negative error number */
+static int coroutine_fn backup_cow_with_bounce_buffer(BackupBlockJob *job,
+                                                      int64_t start,
+                                                      int64_t end,
+                                                      bool is_write_notifier,
+                                                      bool *error_is_read,
+                                                      void **bounce_buffer)
+{
+    int ret;
+    BlockBackend *blk = job->common.blk;
+    int nbytes;
+    int read_flags = is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0;
+    int write_flags = job->serialize_target_writes ? BDRV_REQ_SERIALISING : 0;
+
+    assert(QEMU_IS_ALIGNED(start, job->cluster_size));
+    hbitmap_reset(job->copy_bitmap, start, job->cluster_size);
+    nbytes = MIN(job->cluster_size, job->len - start);
+    if (!*bounce_buffer) {
+        *bounce_buffer = blk_blockalign(blk, job->cluster_size);
+    }
+
+    ret = blk_co_pread(blk, start, nbytes, *bounce_buffer, read_flags);
+    if (ret < 0) {
+        trace_backup_do_cow_read_fail(job, start, ret);
+        if (error_is_read) {
+            *error_is_read = true;
+        }
+        goto fail;
+    }
+
+    if (buffer_is_zero(*bounce_buffer, nbytes)) {
+        ret = blk_co_pwrite_zeroes(job->target, start,
+                                   nbytes, write_flags | BDRV_REQ_MAY_UNMAP);
+    } else {
+        ret = blk_co_pwrite(job->target, start,
+                            nbytes, *bounce_buffer, write_flags |
+                            (job->compress ? BDRV_REQ_WRITE_COMPRESSED : 0));
+    }
+    if (ret < 0) {
+        trace_backup_do_cow_write_fail(job, start, ret);
+        if (error_is_read) {
+            *error_is_read = false;
+        }
+        goto fail;
+    }
+
+    return nbytes;
+fail:
+    hbitmap_set(job->copy_bitmap, start, job->cluster_size);
+    return ret;
+
+}
+
+/* Copy range to target and return the bytes copied. If error occurred, return a
+ * negative error number. */
+static int coroutine_fn backup_cow_with_offload(BackupBlockJob *job,
+                                                int64_t start,
+                                                int64_t end,
+                                                bool is_write_notifier)
+{
+    int ret;
+    int nr_clusters;
+    BlockBackend *blk = job->common.blk;
+    int nbytes;
+    int read_flags = is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0;
+    int write_flags = job->serialize_target_writes ? BDRV_REQ_SERIALISING : 0;
+
+    assert(QEMU_IS_ALIGNED(job->copy_range_size, job->cluster_size));
+    assert(QEMU_IS_ALIGNED(start, job->cluster_size));
+    nbytes = MIN(job->copy_range_size, MIN(end, job->len) - start);
+    nr_clusters = DIV_ROUND_UP(nbytes, job->cluster_size);
+    hbitmap_reset(job->copy_bitmap, start, job->cluster_size * nr_clusters);
+    ret = blk_co_copy_range(blk, start, job->target, start, nbytes,
+                            read_flags, write_flags);
+    if (ret < 0) {
+        trace_backup_do_cow_copy_range_fail(job, start, ret);
+        hbitmap_set(job->copy_bitmap, start, job->cluster_size * nr_clusters);
+        return ret;
+    }
+
+    return nbytes;
 }

 static int coroutine_fn backup_do_cow(BackupBlockJob *job,
                                      int64_t offset, uint64_t bytes,
-                                      bool *error_is_read)
+                                      bool *error_is_read,
+                                      bool is_write_notifier)
 {
+    CowRequest cow_request;
    int ret = 0;
    int64_t start, end; /* bytes */
+    void *bounce_buffer = NULL;
+
+    qemu_co_rwlock_rdlock(&job->flush_rwlock);

    start = QEMU_ALIGN_DOWN(offset, job->cluster_size);
    end = QEMU_ALIGN_UP(bytes + offset, job->cluster_size);

    trace_backup_do_cow_enter(job, start, offset, bytes);

-    ret = block_copy(job->bcs, start, end - start, error_is_read);
+    wait_for_overlapping_requests(job, start, end);
+    cow_request_begin(&cow_request, job, start, end);
+
+    while (start < end) {
+        int64_t dirty_end;
+
+        if (!hbitmap_get(job->copy_bitmap, start)) {
+            trace_backup_do_cow_skip(job, start);
+            start += job->cluster_size;
+            continue; /* already copied */
+        }
+
+        dirty_end = hbitmap_next_zero(job->copy_bitmap, start, (end - start));
+        if (dirty_end < 0) {
+            dirty_end = end;
+        }
+
+        trace_backup_do_cow_process(job, start);
+
+        if (job->use_copy_range) {
+            ret = backup_cow_with_offload(job, start, dirty_end,
+                                          is_write_notifier);
+            if (ret < 0) {
+                job->use_copy_range = false;
+            }
+        }
+        if (!job->use_copy_range) {
+            ret = backup_cow_with_bounce_buffer(job, start, dirty_end,
+                                                is_write_notifier,
+                                                error_is_read, &bounce_buffer);
+        }
+        if (ret < 0) {
+            break;
+        }
+
+        /* Publish progress, guest I/O counts as progress too.  Note that the
+         * offset field is an opaque progress value, it is not a disk offset.
+         */
+        start += ret;
+        job->bytes_read += ret;
+        job_progress_update(&job->common.job, ret);
+        ret = 0;
+    }
+
+    if (bounce_buffer) {
+        qemu_vfree(bounce_buffer);
+    }
+
+    cow_request_end(&cow_request);

    trace_backup_do_cow_return(job, offset, bytes, ret);

+    qemu_co_rwlock_unlock(&job->flush_rwlock);
+
    return ret;
 }

+static int coroutine_fn backup_before_write_notify(
+        NotifierWithReturn *notifier,
+        void *opaque)
+{
+    BackupBlockJob *job = container_of(notifier, BackupBlockJob, before_write);
+    BdrvTrackedRequest *req = opaque;
+
+    assert(req->bs == blk_bs(job->common.blk));
+    assert(QEMU_IS_ALIGNED(req->offset, BDRV_SECTOR_SIZE));
+    assert(QEMU_IS_ALIGNED(req->bytes, BDRV_SECTOR_SIZE));
+
+    return backup_do_cow(job, req->offset, req->bytes, NULL, true);
+}
+
 static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
 {
    BdrvDirtyBitmap *bm;
-    bool sync = (((ret == 0) || (job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS)) \
-                 && (job->bitmap_mode != BITMAP_SYNC_MODE_NEVER));
+    BlockDriverState *bs = blk_bs(job->common.blk);

-    if (sync) {
-        /*
-         * We succeeded, or we always intended to sync the bitmap.
-         * Delete this bitmap and install the child.
-         */
-        bm = bdrv_dirty_bitmap_abdicate(job->sync_bitmap, NULL);
+    if (ret < 0) {
+        /* Merge the successor back into the parent, delete nothing. */
+        bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
+        assert(bm);
    } else {
-        /*
-         * We failed, or we never intended to sync the bitmap anyway.
-         * Merge the successor back into the parent, keeping all data.
-         */
-        bm = bdrv_reclaim_dirty_bitmap(job->sync_bitmap, NULL);
-    }
-
-    assert(bm);
-
-    if (ret < 0 && job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS) {
-        /* If we failed and synced, merge in the bits we didn't copy: */
-        bdrv_dirty_bitmap_merge_internal(bm, job->bcs->copy_bitmap,
-                                         NULL, true);
+        /* Everything is fine, delete this bitmap and install the backup. */
+        bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
+        assert(bm);
    }
 }

@@ -135,8 +304,14 @@ static void backup_abort(Job *job)
 static void backup_clean(Job *job)
 {
    BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);
+    assert(s->target);
+    blk_unref(s->target);
+    s->target = NULL;

-    bdrv_backup_top_drop(s->backup_top);
+    if (s->copy_bitmap) {
+        hbitmap_free(s->copy_bitmap);
+        s->copy_bitmap = NULL;
+    }
 }

 void backup_do_checkpoint(BlockJob *job, Error **errp)
@@ -151,7 +326,22 @@ void backup_do_checkpoint(BlockJob *job, Error **errp)
        return;
    }

-    bdrv_set_dirty_bitmap(backup_job->bcs->copy_bitmap, 0, backup_job->len);
+    hbitmap_set(backup_job->copy_bitmap, 0, backup_job->len);
+}
+
+static void backup_drain(BlockJob *job)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+
+    /* Need to keep a reference in case blk_drain triggers execution
+     * of backup_complete...
+     */
+    if (s->target) {
+        BlockBackend *target = s->target;
+        blk_ref(target);
+        blk_drain(target);
+        blk_unref(target);
+    }
 }

 static BlockErrorAction backup_error_action(BackupBlockJob *job,
@@ -174,10 +364,8 @@ static bool coroutine_fn yield_and_check(BackupBlockJob *job)
        return true;
    }

-    /*
-     * We need to yield even for delay_ns = 0 so that bdrv_drain_all() can
-     * return. Without a yield, the VM would not reboot.
-     */
+    /* We need to yield even for delay_ns = 0 so that bdrv_drain_all() can
+     * return. Without a yield, the VM would not reboot. */
    delay_ns = block_job_ratelimit_get_delay(&job->common, job->bytes_read);
    job->bytes_read = 0;
    job_sleep_ns(&job->common.job, delay_ns);
@@ -189,102 +377,117 @@ static bool coroutine_fn yield_and_check(BackupBlockJob *job)
    return false;
 }

+static bool bdrv_is_unallocated_range(BlockDriverState *bs,
+                                      int64_t offset, int64_t bytes)
+{
+    int64_t end = offset + bytes;
+
+    while (offset < end && !bdrv_is_allocated(bs, offset, bytes, &bytes)) {
+        if (bytes == 0) {
+            return true;
+        }
+        offset += bytes;
+        bytes = end - offset;
+    }
+
+    return offset >= end;
+}
+
 static int coroutine_fn backup_loop(BackupBlockJob *job)
 {
+    int ret;
    bool error_is_read;
    int64_t offset;
-    BdrvDirtyBitmapIter *bdbi;
-    int ret = 0;
+    HBitmapIter hbi;
+    BlockDriverState *bs = blk_bs(job->common.blk);
+
+    hbitmap_iter_init(&hbi, job->copy_bitmap, 0);
+    while ((offset = hbitmap_iter_next(&hbi)) != -1) {
+        if (job->sync_mode == MIRROR_SYNC_MODE_TOP &&
+            bdrv_is_unallocated_range(bs, offset, job->cluster_size))
+        {
+            hbitmap_reset(job->copy_bitmap, offset, job->cluster_size);
+            continue;
+        }

-    bdbi = bdrv_dirty_iter_new(job->bcs->copy_bitmap);
-    while ((offset = bdrv_dirty_iter_next(bdbi)) != -1) {
        do {
            if (yield_and_check(job)) {
-                goto out;
+                return 0;
            }
-            ret = backup_do_cow(job, offset, job->cluster_size, &error_is_read);
+            ret = backup_do_cow(job, offset,
+                                job->cluster_size, &error_is_read, false);
            if (ret < 0 && backup_error_action(job, error_is_read, -ret) ==
                           BLOCK_ERROR_ACTION_REPORT)
            {
-                goto out;
+                return ret;
            }
        } while (ret < 0);
    }

- out:
-    bdrv_dirty_iter_free(bdbi);
-    return ret;
+    return 0;
 }

-static void backup_init_copy_bitmap(BackupBlockJob *job)
+/* init copy_bitmap from sync_bitmap */
+static void backup_incremental_init_copy_bitmap(BackupBlockJob *job)
 {
-    bool ret;
-    uint64_t estimate;
+    uint64_t offset = 0;
+    uint64_t bytes = job->len;

-    if (job->sync_mode == MIRROR_SYNC_MODE_BITMAP) {
-        ret = bdrv_dirty_bitmap_merge_internal(job->bcs->copy_bitmap,
-                                               job->sync_bitmap,
-                                               NULL, true);
-        assert(ret);
-    } else {
-        if (job->sync_mode == MIRROR_SYNC_MODE_TOP) {
-            /*
-             * We can't hog the coroutine to initialize this thoroughly.
-             * Set a flag and resume work when we are able to yield safely.
-             */
-            job->bcs->skip_unallocated = true;
+    while (bdrv_dirty_bitmap_next_dirty_area(job->sync_bitmap,
+                                             &offset, &bytes))
+    {
+        hbitmap_set(job->copy_bitmap, offset, bytes);
+
+        offset += bytes;
+        if (offset >= job->len) {
+            break;
        }
-        bdrv_set_dirty_bitmap(job->bcs->copy_bitmap, 0, job->len);
+        bytes = job->len - offset;
    }

-    estimate = bdrv_get_dirty_count(job->bcs->copy_bitmap);
-    job_progress_set_remaining(&job->common.job, estimate);
+    /* TODO job_progress_set_remaining() would make more sense */
+    job_progress_update(&job->common.job,
+        job->len - hbitmap_count(job->copy_bitmap));
 }

 static int coroutine_fn backup_run(Job *job, Error **errp)
 {
    BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);
+    BlockDriverState *bs = blk_bs(s->common.blk);
    int ret = 0;

-    backup_init_copy_bitmap(s);
+    QLIST_INIT(&s->inflight_reqs);
+    qemu_co_rwlock_init(&s->flush_rwlock);

-    if (s->sync_mode == MIRROR_SYNC_MODE_TOP) {
-        int64_t offset = 0;
-        int64_t count;
+    job_progress_set_remaining(job, s->len);

-        for (offset = 0; offset < s->len; ) {
-            if (yield_and_check(s)) {
-                ret = -ECANCELED;
-                goto out;
-            }
-
-            ret = block_copy_reset_unallocated(s->bcs, offset, &count);
-            if (ret < 0) {
-                goto out;
-            }
-
-            offset += count;
-        }
-        s->bcs->skip_unallocated = false;
+    if (s->sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
+        backup_incremental_init_copy_bitmap(s);
+    } else {
+        hbitmap_set(s->copy_bitmap, 0, s->len);
    }

+    s->before_write.notify = backup_before_write_notify;
+    bdrv_add_before_write_notifier(bs, &s->before_write);
+
    if (s->sync_mode == MIRROR_SYNC_MODE_NONE) {
-        /*
-         * All bits are set in copy_bitmap to allow any cluster to be copied.
-         * This does not actually require them to be copied.
-         */
+        /* All bits are set in copy_bitmap to allow any cluster to be copied.
+         * This does not actually require them to be copied. */
        while (!job_is_cancelled(job)) {
-            /*
-             * Yield until the job is cancelled.  We just let our before_write
-             * notify callback service CoW requests.
-             */
+            /* Yield until the job is cancelled.  We just let our before_write
+             * notify callback service CoW requests. */
            job_yield(job);
        }
    } else {
        ret = backup_loop(s);
    }

- out:
+    notifier_with_return_remove(&s->before_write);
+
+    /* wait until pending backup_do_cow() calls have completed */
+    qemu_co_rwlock_wrlock(&s->flush_rwlock);
+    qemu_co_rwlock_unlock(&s->flush_rwlock);
+
    return ret;
 }

@@ -294,11 +497,13 @@ static const BlockJobDriver backup_job_driver = {
        .job_type               = JOB_TYPE_BACKUP,
        .free                   = block_job_free,
        .user_resume            = block_job_user_resume,
+        .drain                  = block_job_drain,
        .run                    = backup_run,
        .commit                 = backup_commit,
        .abort                  = backup_abort,
        .clean                  = backup_clean,
-    }
+    },
+    .drain                  = backup_drain,
 };

 static int64_t backup_calculate_cluster_size(BlockDriverState *target,
@@ -340,9 +545,7 @@ static int64_t backup_calculate_cluster_size(BlockDriverState *target,
 BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
                  BlockDriverState *target, int64_t speed,
                  MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap,
-                  BitmapSyncMode bitmap_mode,
                  bool compress,
-                  const char *filter_node_name,
                  BlockdevOnError on_source_error,
                  BlockdevOnError on_target_error,
                  int creation_flags,
@@ -351,18 +554,13 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
 {
    int64_t len;
    BackupBlockJob *job = NULL;
+    int ret;
    int64_t cluster_size;
-    BdrvRequestFlags write_flags;
-    BlockDriverState *backup_top = NULL;
-    BlockCopyState *bcs = NULL;
+    HBitmap *copy_bitmap = NULL;

    assert(bs);
    assert(target);

-    /* QMP interface protects us from these cases */
-    assert(sync_mode != MIRROR_SYNC_MODE_INCREMENTAL);
-    assert(sync_bitmap || sync_mode != MIRROR_SYNC_MODE_BITMAP);
-
    if (bs == target) {
        error_setg(errp, "Source and target cannot be the same");
        return NULL;
@@ -380,7 +578,7 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
        return NULL;
    }

-    if (compress && !block_driver_can_compress(target->drv)) {
+    if (compress && target->drv->bdrv_co_pwritev_compressed == NULL) {
        error_setg(errp, "Compression is not supported for this drive %s",
                   bdrv_get_device_name(target));
        return NULL;
@@ -394,17 +592,23 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
        return NULL;
    }

-    if (sync_bitmap) {
-        /* If we need to write to this bitmap, check that we can: */
-        if (bitmap_mode != BITMAP_SYNC_MODE_NEVER &&
-            bdrv_dirty_bitmap_check(sync_bitmap, BDRV_BITMAP_DEFAULT, errp)) {
+    if (sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
+        if (!sync_bitmap) {
+            error_setg(errp, "must provide a valid bitmap name for "
+                             "\"incremental\" sync mode");
            return NULL;
        }

        /* Create a new bitmap, and freeze/disable this one. */
-        if (bdrv_dirty_bitmap_create_successor(sync_bitmap, errp) < 0) {
+        if (bdrv_dirty_bitmap_create_successor(bs, sync_bitmap, errp) < 0) {
            return NULL;
        }
+    } else if (sync_bitmap) {
+        error_setg(errp,
+                   "a sync_bitmap was provided to backup_run, "
+                   "but received an incompatible sync_mode (%s)",
+                   MirrorSyncMode_str(sync_mode));
+        return NULL;
    }

    len = bdrv_getlength(bs);
@@ -419,63 +623,72 @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
        goto error;
    }

-    /*
-     * If source is in backing chain of target assume that target is going to be
-     * used for "image fleecing", i.e. it should represent a kind of snapshot of
-     * source at backup-start point in time. And target is going to be read by
-     * somebody (for example, used as NBD export) during backup job.
-     *
-     * In this case, we need to add BDRV_REQ_SERIALISING write flag to avoid
-     * intersection of backup writes and third party reads from target,
-     * otherwise reading from target we may occasionally read already updated by
-     * guest data.
-     *
-     * For more information see commit f8d59dfb40bb and test
-     * tests/qemu-iotests/222
-     */
-    write_flags = (bdrv_chain_contains(target, bs) ? BDRV_REQ_SERIALISING : 0) |
-                  (compress ? BDRV_REQ_WRITE_COMPRESSED : 0),
-
-    backup_top = bdrv_backup_top_append(bs, target, filter_node_name,
-                                        cluster_size, write_flags, &bcs, errp);
-    if (!backup_top) {
-        goto error;
-    }
+    copy_bitmap = hbitmap_alloc(len, ctz32(cluster_size));

    /* job->len is fixed, so we can't allow resize */
-    job = block_job_create(job_id, &backup_job_driver, txn, backup_top,
-                           0, BLK_PERM_ALL,
+    job = block_job_create(job_id, &backup_job_driver, txn, bs,
+                           BLK_PERM_CONSISTENT_READ,
+                           BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE |
+                           BLK_PERM_WRITE_UNCHANGED | BLK_PERM_GRAPH_MOD,
                           speed, creation_flags, cb, opaque, errp);
    if (!job) {
        goto error;
    }

-    job->backup_top = backup_top;
-    job->source_bs = bs;
+    /* The target must match the source in size, so no resize here either */
+    job->target = blk_new(job->common.job.aio_context,
+                          BLK_PERM_WRITE,
+                          BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE |
+                          BLK_PERM_WRITE_UNCHANGED | BLK_PERM_GRAPH_MOD);
+    ret = blk_insert_bs(job->target, target, errp);
+    if (ret < 0) {
+        goto error;
+    }
+
    job->on_source_error = on_source_error;
    job->on_target_error = on_target_error;
    job->sync_mode = sync_mode;
-    job->sync_bitmap = sync_bitmap;
-    job->bitmap_mode = bitmap_mode;
-    job->bcs = bcs;
+    job->sync_bitmap = sync_mode == MIRROR_SYNC_MODE_INCREMENTAL ?
+                       sync_bitmap : NULL;
+    job->compress = compress;
+
+    /* Detect image-fleecing (and similar) schemes */
+    job->serialize_target_writes = bdrv_chain_contains(target, bs);
    job->cluster_size = cluster_size;
-    job->len = len;
+    job->copy_bitmap = copy_bitmap;
+    copy_bitmap = NULL;
+    job->copy_range_size = MIN_NON_ZERO(blk_get_max_transfer(job->common.blk),
+                                        blk_get_max_transfer(job->target));
+    job->copy_range_size = QEMU_ALIGN_DOWN(job->copy_range_size,
+                                           job->cluster_size);
+    /*
+     * Set use_copy_range, consider the following:
+     * 1. Compression is not supported for copy_range.
+     * 2. copy_range does not respect max_transfer (it's a TODO), so we factor
+     *    that in here. If max_transfer is smaller than the job->cluster_size,
+     *    we do not use copy_range (in that case it's zero after aligning down
+     *    above).
+     */
+    job->use_copy_range = !compress && job->copy_range_size > 0;

-    block_copy_set_callbacks(bcs, backup_progress_bytes_callback,
-                             backup_progress_reset_callback, job);
-
-    /* Required permissions are already taken by backup-top target */
+    /* Required permissions are already taken with target's blk_new() */
    block_job_add_bdrv(&job->common, "target", target, 0, BLK_PERM_ALL,
                       &error_abort);
+    job->len = len;

    return &job->common;

 error:
-    if (sync_bitmap) {
-        bdrv_reclaim_dirty_bitmap(sync_bitmap, NULL);
+    if (copy_bitmap) {
+        assert(!job || !job->copy_bitmap);
+        hbitmap_free(copy_bitmap);
    }
-    if (backup_top) {
-        bdrv_backup_top_drop(backup_top);
+    if (sync_bitmap) {
+        bdrv_reclaim_dirty_bitmap(bs, sync_bitmap, NULL);
+    }
+    if (job) {
+        backup_clean(&job->common.job);
+        job_early_fail(&job->common.job);
    }

    return NULL;
@@ -126,12 +126,6 @@ static int coroutine_fn blkreplay_co_flush(BlockDriverState *bs)
    return ret;
 }

-static int blkreplay_snapshot_goto(BlockDriverState *bs,
-                                   const char *snapshot_id)
-{
-    return bdrv_snapshot_goto(bs->file->bs, snapshot_id, NULL);
-}
-
 static BlockDriver bdrv_blkreplay = {
    .format_name            = "blkreplay",
    .instance_size          = 0,
@@ -146,8 +140,6 @@ static BlockDriver bdrv_blkreplay = {
    .bdrv_co_pwrite_zeroes  = blkreplay_co_pwrite_zeroes,
    .bdrv_co_pdiscard       = blkreplay_co_pdiscard,
    .bdrv_co_flush          = blkreplay_co_flush,
-
-    .bdrv_snapshot_goto     = blkreplay_snapshot_goto,
 };

 static void bdrv_blkreplay_init(void)
@@ -15,15 +15,11 @@
 #include "block/block_int.h"
 #include "block/blockjob.h"
 #include "block/throttle-groups.h"
-#include "hw/qdev-core.h"
 #include "sysemu/blockdev.h"
-#include "sysemu/runstate.h"
 #include "sysemu/sysemu.h"
-#include "sysemu/replay.h"
 #include "qapi/error.h"
 #include "qapi/qapi-events-block.h"
 #include "qemu/id.h"
-#include "qemu/main-loop.h"
 #include "qemu/option.h"
 #include "trace.h"
 #include "migration/misc.h"
@@ -83,9 +79,6 @@ struct BlockBackend {
    QLIST_HEAD(, BlockBackendAioNotifier) aio_notifiers;

    int quiesce_counter;
-    CoQueue queued_requests;
-    bool disable_request_queuing;
-
    VMChangeStateEntry *vmsh;
    bool force_allow_inactivate;

@@ -346,7 +339,6 @@ BlockBackend *blk_new(AioContext *ctx, uint64_t perm, uint64_t shared_perm)

    block_acct_init(&blk->stats);

-    qemu_co_queue_init(&blk->queued_requests);
    notifier_list_init(&blk->remove_bs_notifiers);
    notifier_list_init(&blk->insert_bs_notifiers);
    QLIST_INIT(&blk->aio_notifiers);
@@ -1104,11 +1096,6 @@ void blk_set_allow_aio_context_change(BlockBackend *blk, bool allow)
    blk->allow_aio_context_change = allow;
 }

-void blk_set_disable_request_queuing(BlockBackend *blk, bool disable)
-{
-    blk->disable_request_queuing = disable;
-}
-
 static int blk_check_byte_request(BlockBackend *blk, int64_t offset,
                                  size_t size)
 {
@@ -1140,24 +1127,13 @@ static int blk_check_byte_request(BlockBackend *blk, int64_t offset,
    return 0;
 }

-static void coroutine_fn blk_wait_while_drained(BlockBackend *blk)
-{
-    if (blk->quiesce_counter && !blk->disable_request_queuing) {
-        qemu_co_queue_wait(&blk->queued_requests, NULL);
-    }
-}
-
 int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
                               unsigned int bytes, QEMUIOVector *qiov,
                               BdrvRequestFlags flags)
 {
    int ret;
-    BlockDriverState *bs;
+    BlockDriverState *bs = blk_bs(blk);

-    blk_wait_while_drained(blk);
-
-    /* Call blk_bs() only after waiting, the graph may have changed */
-    bs = blk_bs(blk);
    trace_blk_co_preadv(blk, bs, offset, bytes, flags);

    ret = blk_check_byte_request(blk, offset, bytes);
@@ -1178,18 +1154,13 @@ int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
    return ret;
 }

-int coroutine_fn blk_co_pwritev_part(BlockBackend *blk, int64_t offset,
-                                     unsigned int bytes,
-                                     QEMUIOVector *qiov, size_t qiov_offset,
-                                     BdrvRequestFlags flags)
+int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
+                                unsigned int bytes, QEMUIOVector *qiov,
+                                BdrvRequestFlags flags)
 {
    int ret;
-    BlockDriverState *bs;
+    BlockDriverState *bs = blk_bs(blk);

-    blk_wait_while_drained(blk);
-
-    /* Call blk_bs() only after waiting, the graph may have changed */
-    bs = blk_bs(blk);
    trace_blk_co_pwritev(blk, bs, offset, bytes, flags);

    ret = blk_check_byte_request(blk, offset, bytes);
@@ -1208,19 +1179,11 @@ int coroutine_fn blk_co_pwritev_part(BlockBackend *blk, int64_t offset,
        flags |= BDRV_REQ_FUA;
    }

-    ret = bdrv_co_pwritev_part(blk->root, offset, bytes, qiov, qiov_offset,
-                               flags);
+    ret = bdrv_co_pwritev(blk->root, offset, bytes, qiov, flags);
    bdrv_dec_in_flight(bs);
    return ret;
 }

-int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
-                                unsigned int bytes, QEMUIOVector *qiov,
-                                BdrvRequestFlags flags)
-{
-    return blk_co_pwritev_part(blk, offset, bytes, qiov, 0, flags);
-}
-
 typedef struct BlkRwCo {
    BlockBackend *blk;
    int64_t offset;
@@ -1274,6 +1237,22 @@ static int blk_prw(BlockBackend *blk, int64_t offset, uint8_t *buf,
    return rwco.ret;
 }

+int blk_pread_unthrottled(BlockBackend *blk, int64_t offset, uint8_t *buf,
+                          int count)
+{
+    int ret;
+
+    ret = blk_check_byte_request(blk, offset, count);
+    if (ret < 0) {
+        return ret;
+    }
+
+    blk_root_drained_begin(blk->root);
+    ret = blk_pread(blk, offset, buf, count);
+    blk_root_drained_end(blk->root, NULL);
+    return ret;
+}
+
 int blk_pwrite_zeroes(BlockBackend *blk, int64_t offset,
                      int bytes, BdrvRequestFlags flags)
 {
@@ -1317,8 +1296,7 @@ BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
    acb->blk = blk;
    acb->ret = ret;

-    replay_bh_schedule_oneshot_event(blk_get_aio_context(blk),
-                                     error_callback_bh, acb);
+    aio_bh_schedule_oneshot(blk_get_aio_context(blk), error_callback_bh, acb);
    return &acb->common;
 }

@@ -1374,8 +1352,8 @@ static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset, int bytes,

    acb->has_returned = true;
    if (acb->rwco.ret != NOT_DONE) {
-        replay_bh_schedule_oneshot_event(blk_get_aio_context(blk),
-                                         blk_aio_complete_bh, acb);
+        aio_bh_schedule_oneshot(blk_get_aio_context(blk),
+                                blk_aio_complete_bh, acb);
    }

    return &acb->common;
@@ -1387,12 +1365,6 @@ static void blk_aio_read_entry(void *opaque)
    BlkRwCo *rwco = &acb->rwco;
    QEMUIOVector *qiov = rwco->iobuf;

-    if (rwco->blk->quiesce_counter) {
-        blk_dec_in_flight(rwco->blk);
-        blk_wait_while_drained(rwco->blk);
-        blk_inc_in_flight(rwco->blk);
-    }
-
    assert(qiov->size == acb->bytes);
    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, acb->bytes,
                              qiov, rwco->flags);
@@ -1405,12 +1377,6 @@ static void blk_aio_write_entry(void *opaque)
    BlkRwCo *rwco = &acb->rwco;
    QEMUIOVector *qiov = rwco->iobuf;

-    if (rwco->blk->quiesce_counter) {
-        blk_dec_in_flight(rwco->blk);
-        blk_wait_while_drained(rwco->blk);
-        blk_inc_in_flight(rwco->blk);
-    }
-
    assert(!qiov || qiov->size == acb->bytes);
    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, acb->bytes,
                               qiov, rwco->flags);
@@ -1532,8 +1498,6 @@ void blk_aio_cancel_async(BlockAIOCB *acb)

 int blk_co_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
 {
-    blk_wait_while_drained(blk);
-
    if (!blk_is_available(blk)) {
        return -ENOMEDIUM;
    }
@@ -1574,11 +1538,7 @@ BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,

 int blk_co_pdiscard(BlockBackend *blk, int64_t offset, int bytes)
 {
-    int ret;
-
-    blk_wait_while_drained(blk);
-
-    ret = blk_check_byte_request(blk, offset, bytes);
+    int ret = blk_check_byte_request(blk, offset, bytes);
    if (ret < 0) {
        return ret;
    }
@@ -1588,8 +1548,6 @@ int blk_co_pdiscard(BlockBackend *blk, int64_t offset, int bytes)

 int blk_co_flush(BlockBackend *blk)
 {
-    blk_wait_while_drained(blk);
-
    if (!blk_is_available(blk)) {
        return -ENOMEDIUM;
    }
@@ -2072,15 +2030,15 @@ int blk_pwrite_compressed(BlockBackend *blk, int64_t offset, const void *buf,
                   BDRV_REQ_WRITE_COMPRESSED);
 }

-int blk_truncate(BlockBackend *blk, int64_t offset, bool exact,
-                 PreallocMode prealloc, Error **errp)
+int blk_truncate(BlockBackend *blk, int64_t offset, PreallocMode prealloc,
+                 Error **errp)
 {
    if (!blk_is_available(blk)) {
        error_setg(errp, "No medium inserted");
        return -ENOMEDIUM;
    }

-    return bdrv_truncate(blk->root, offset, exact, prealloc, errp);
+    return bdrv_truncate(blk->root, offset, prealloc, errp);
 }

 static void blk_pdiscard_entry(void *opaque)
@@ -2290,9 +2248,6 @@ static void blk_root_drained_end(BdrvChild *child, int *drained_end_counter)
        if (blk->dev_ops && blk->dev_ops->drained_end) {
            blk->dev_ops->drained_end(blk->dev_opaque);
        }
-        while (qemu_co_enter_next(&blk->queued_requests, NULL)) {
-            /* Resume all queued requests */
-        }
    }
 }

@@ -1,351 +0,0 @@
-/*
- * block_copy API
- *
- * Copyright (C) 2013 Proxmox Server Solutions
- * Copyright (c) 2019 Virtuozzo International GmbH.
- *
- * Authors:
- *  Dietmar Maurer (dietmar@proxmox.com)
- *  Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- */
-
-#include "qemu/osdep.h"
-
-#include "trace.h"
-#include "qapi/error.h"
-#include "block/block-copy.h"
-#include "sysemu/block-backend.h"
-#include "qemu/units.h"
-
-#define BLOCK_COPY_MAX_COPY_RANGE (16 * MiB)
-#define BLOCK_COPY_MAX_BUFFER (1 * MiB)
-#define BLOCK_COPY_MAX_MEM (128 * MiB)
-
-static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
-                                                       int64_t start,
-                                                       int64_t end)
-{
-    BlockCopyInFlightReq *req;
-    bool waited;
-
-    do {
-        waited = false;
-        QLIST_FOREACH(req, &s->inflight_reqs, list) {
-            if (end > req->start_byte && start < req->end_byte) {
-                qemu_co_queue_wait(&req->wait_queue, NULL);
-                waited = true;
-                break;
-            }
-        }
-    } while (waited);
-}
-
-static void block_copy_inflight_req_begin(BlockCopyState *s,
-                                          BlockCopyInFlightReq *req,
-                                          int64_t start, int64_t end)
-{
-    req->start_byte = start;
-    req->end_byte = end;
-    qemu_co_queue_init(&req->wait_queue);
-    QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
-}
-
-static void coroutine_fn block_copy_inflight_req_end(BlockCopyInFlightReq *req)
-{
-    QLIST_REMOVE(req, list);
-    qemu_co_queue_restart_all(&req->wait_queue);
-}
-
-void block_copy_state_free(BlockCopyState *s)
-{
-    if (!s) {
-        return;
-    }
-
-    bdrv_release_dirty_bitmap(s->copy_bitmap);
-    shres_destroy(s->mem);
-    g_free(s);
-}
-
-BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
-                                     int64_t cluster_size,
-                                     BdrvRequestFlags write_flags, Error **errp)
-{
-    BlockCopyState *s;
-    BdrvDirtyBitmap *copy_bitmap;
-    uint32_t max_transfer =
-            MIN_NON_ZERO(INT_MAX,
-                         MIN_NON_ZERO(source->bs->bl.max_transfer,
-                                      target->bs->bl.max_transfer));
-
-    copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL,
-                                           errp);
-    if (!copy_bitmap) {
-        return NULL;
-    }
-    bdrv_disable_dirty_bitmap(copy_bitmap);
-
-    s = g_new(BlockCopyState, 1);
-    *s = (BlockCopyState) {
-        .source = source,
-        .target = target,
-        .copy_bitmap = copy_bitmap,
-        .cluster_size = cluster_size,
-        .len = bdrv_dirty_bitmap_size(copy_bitmap),
-        .write_flags = write_flags,
-        .mem = shres_create(BLOCK_COPY_MAX_MEM),
-    };
-
-    if (max_transfer < cluster_size) {
-        /*
-         * copy_range does not respect max_transfer. We don't want to bother
-         * with requests smaller than block-copy cluster size, so fallback to
-         * buffered copying (read and write respect max_transfer on their
-         * behalf).
-         */
-        s->use_copy_range = false;
-        s->copy_size = cluster_size;
-    } else if (write_flags & BDRV_REQ_WRITE_COMPRESSED) {
-        /* Compression supports only cluster-size writes and no copy-range. */
-        s->use_copy_range = false;
-        s->copy_size = cluster_size;
-    } else {
-        /*
-         * copy_range does not respect max_transfer (it's a TODO), so we factor
-         * that in here.
-         */
-        s->use_copy_range = true;
-        s->copy_size = MIN(MAX(cluster_size, BLOCK_COPY_MAX_COPY_RANGE),
-                           QEMU_ALIGN_DOWN(max_transfer, cluster_size));
-    }
-
-    QLIST_INIT(&s->inflight_reqs);
-
-    return s;
-}
-
-void block_copy_set_callbacks(
-        BlockCopyState *s,
-        ProgressBytesCallbackFunc progress_bytes_callback,
-        ProgressResetCallbackFunc progress_reset_callback,
-        void *progress_opaque)
-{
-    s->progress_bytes_callback = progress_bytes_callback;
-    s->progress_reset_callback = progress_reset_callback;
-    s->progress_opaque = progress_opaque;
-}
-
-/*
- * block_copy_do_copy
- *
- * Do copy of cluser-aligned chunk. @end is allowed to exceed s->len only to
- * cover last cluster when s->len is not aligned to clusters.
- *
- * No sync here: nor bitmap neighter intersecting requests handling, only copy.
- *
- * Returns 0 on success.
- */
-static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
-                                           int64_t start, int64_t end,
-                                           bool *error_is_read)
-{
-    int ret;
-    int nbytes = MIN(end, s->len) - start;
-    void *bounce_buffer = NULL;
-
-    assert(QEMU_IS_ALIGNED(start, s->cluster_size));
-    assert(QEMU_IS_ALIGNED(end, s->cluster_size));
-    assert(end < s->len || end == QEMU_ALIGN_UP(s->len, s->cluster_size));
-
-    if (s->use_copy_range) {
-        ret = bdrv_co_copy_range(s->source, start, s->target, start, nbytes,
-                                 0, s->write_flags);
-        if (ret < 0) {
-            trace_block_copy_copy_range_fail(s, start, ret);
-            s->use_copy_range = false;
-            s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
-            /* Fallback to read+write with allocated buffer */
-        } else {
-            goto out;
-        }
-    }
-
-    /*
-     * In case of failed copy_range request above, we may proceed with buffered
-     * request larger than BLOCK_COPY_MAX_BUFFER. Still, further requests will
-     * be properly limited, so don't care too much.
-     */
-
-    bounce_buffer = qemu_blockalign(s->source->bs, nbytes);
-
-    ret = bdrv_co_pread(s->source, start, nbytes, bounce_buffer, 0);
-    if (ret < 0) {
-        trace_block_copy_read_fail(s, start, ret);
-        if (error_is_read) {
-            *error_is_read = true;
-        }
-        goto out;
-    }
-
-    ret = bdrv_co_pwrite(s->target, start, nbytes, bounce_buffer,
-                         s->write_flags);
-    if (ret < 0) {
-        trace_block_copy_write_fail(s, start, ret);
-        if (error_is_read) {
-            *error_is_read = false;
-        }
-        goto out;
-    }
-
-out:
-    qemu_vfree(bounce_buffer);
-
-    return ret;
-}
-
-/*
- * Check if the cluster starting at offset is allocated or not.
- * return via pnum the number of contiguous clusters sharing this allocation.
- */
-static int block_copy_is_cluster_allocated(BlockCopyState *s, int64_t offset,
-                                           int64_t *pnum)
-{
-    BlockDriverState *bs = s->source->bs;
-    int64_t count, total_count = 0;
-    int64_t bytes = s->len - offset;
-    int ret;
-
-    assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
-
-    while (true) {
-        ret = bdrv_is_allocated(bs, offset, bytes, &count);
-        if (ret < 0) {
-            return ret;
-        }
-
-        total_count += count;
-
-        if (ret || count == 0) {
-            /*
-             * ret: partial segment(s) are considered allocated.
-             * otherwise: unallocated tail is treated as an entire segment.
-             */
-            *pnum = DIV_ROUND_UP(total_count, s->cluster_size);
-            return ret;
-        }
-
-        /* Unallocated segment(s) with uncertain following segment(s) */
-        if (total_count >= s->cluster_size) {
-            *pnum = total_count / s->cluster_size;
-            return 0;
-        }
-
-        offset += count;
-        bytes -= count;
-    }
-}
-
-/*
- * Reset bits in copy_bitmap starting at offset if they represent unallocated
- * data in the image. May reset subsequent contiguous bits.
- * @return 0 when the cluster at @offset was unallocated,
- *         1 otherwise, and -ret on error.
- */
-int64_t block_copy_reset_unallocated(BlockCopyState *s,
-                                     int64_t offset, int64_t *count)
-{
-    int ret;
-    int64_t clusters, bytes;
-
-    ret = block_copy_is_cluster_allocated(s, offset, &clusters);
-    if (ret < 0) {
-        return ret;
-    }
-
-    bytes = clusters * s->cluster_size;
-
-    if (!ret) {
-        bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
-        s->progress_reset_callback(s->progress_opaque);
-    }
-
-    *count = bytes;
-    return ret;
-}
-
-int coroutine_fn block_copy(BlockCopyState *s,
-                            int64_t start, uint64_t bytes,
-                            bool *error_is_read)
-{
-    int ret = 0;
-    int64_t end = bytes + start; /* bytes */
-    int64_t status_bytes;
-    BlockCopyInFlightReq req;
-
-    /*
-     * block_copy() user is responsible for keeping source and target in same
-     * aio context
-     */
-    assert(bdrv_get_aio_context(s->source->bs) ==
-           bdrv_get_aio_context(s->target->bs));
-
-    assert(QEMU_IS_ALIGNED(start, s->cluster_size));
-    assert(QEMU_IS_ALIGNED(end, s->cluster_size));
-
-    block_copy_wait_inflight_reqs(s, start, bytes);
-    block_copy_inflight_req_begin(s, &req, start, end);
-
-    while (start < end) {
-        int64_t next_zero, chunk_end;
-
-        if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) {
-            trace_block_copy_skip(s, start);
-            start += s->cluster_size;
-            continue; /* already copied */
-        }
-
-        chunk_end = MIN(end, start + s->copy_size);
-
-        next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, start,
-                                                chunk_end - start);
-        if (next_zero >= 0) {
-            assert(next_zero > start); /* start is dirty */
-            assert(next_zero < chunk_end); /* no need to do MIN() */
-            chunk_end = next_zero;
-        }
-
-        if (s->skip_unallocated) {
-            ret = block_copy_reset_unallocated(s, start, &status_bytes);
-            if (ret == 0) {
-                trace_block_copy_skip_range(s, start, status_bytes);
-                start += status_bytes;
-                continue;
-            }
-            /* Clamp to known allocated region */
-            chunk_end = MIN(chunk_end, start + status_bytes);
-        }
-
-        trace_block_copy_process(s, start);
-
-        bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
-
-        co_get_from_shres(s->mem, chunk_end - start);
-        ret = block_copy_do_copy(s, start, chunk_end, error_is_read);
-        co_put_to_shres(s->mem, chunk_end - start);
-        if (ret < 0) {
-            bdrv_set_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
-            break;
-        }
-
-        s->progress_bytes_callback(chunk_end - start, s->progress_opaque);
-        start = chunk_end;
-        ret = 0;
-    }
-
-    block_copy_inflight_req_end(&req);
-
-    return ret;
-}
@@ -248,8 +248,8 @@ bochs_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
    QEMUIOVector local_qiov;
    int ret;

-    assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
-    assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

    qemu_iovec_init(&local_qiov, qiov->niov);
    qemu_co_mutex_lock(&s->lock);
@@ -253,8 +253,8 @@ cloop_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
    int nb_sectors = bytes >> BDRV_SECTOR_BITS;
    int ret, i;

-    assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
-    assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

    qemu_co_mutex_lock(&s->lock);

@@ -155,7 +155,7 @@ static int coroutine_fn commit_run(Job *job, Error **errp)
    }

    if (base_len < len) {
-        ret = blk_truncate(s->base, len, false, PREALLOC_MODE_OFF, NULL);
+        ret = blk_truncate(s->base, len, PREALLOC_MODE_OFF, NULL);
        if (ret) {
            goto out;
        }
@@ -216,6 +216,7 @@ static const BlockJobDriver commit_job_driver = {
        .job_type      = JOB_TYPE_COMMIT,
        .free          = block_job_free,
        .user_resume   = block_job_user_resume,
+        .drain         = block_job_drain,
        .run           = commit_run,
        .prepare       = commit_prepare,
        .abort         = commit_abort,
@@ -349,7 +350,6 @@ void commit_start(const char *job_id, BlockDriverState *bs,
    if (ret < 0) {
        goto fail;
    }
-    blk_set_disable_request_queuing(s->base, true);
    s->base_bs = base;

    /* Required permissions are already taken with block_job_add_bdrv() */
@@ -358,7 +358,6 @@ void commit_start(const char *job_id, BlockDriverState *bs,
    if (ret < 0) {
        goto fail;
    }
-    blk_set_disable_request_queuing(s->top, true);

    s->backing_file_str = g_strdup(backing_file_str);
    s->on_error = on_error;
@@ -471,8 +470,7 @@ int bdrv_commit(BlockDriverState *bs)
     * grow the backing file image if possible.  If not possible,
     * we must return an error */
    if (length > backing_length) {
-        ret = blk_truncate(backing, length, false, PREALLOC_MODE_OFF,
-                           &local_err);
+        ret = blk_truncate(backing, length, PREALLOC_MODE_OFF, &local_err);
        if (ret < 0) {
            error_report_err(local_err);
            goto ro_cleanup;
@@ -73,6 +73,13 @@ static int64_t cor_getlength(BlockDriverState *bs)
 }


+static int coroutine_fn cor_co_truncate(BlockDriverState *bs, int64_t offset,
+                                        PreallocMode prealloc, Error **errp)
+{
+    return bdrv_co_truncate(bs->file, offset, prealloc, errp);
+}
+
+
 static int coroutine_fn cor_co_preadv(BlockDriverState *bs,
                                      uint64_t offset, uint64_t bytes,
                                      QEMUIOVector *qiov, int flags)
@@ -132,6 +139,7 @@ static BlockDriver bdrv_copy_on_read = {
    .bdrv_child_perm                    = cor_child_perm,

    .bdrv_getlength                     = cor_getlength,
+    .bdrv_co_truncate                   = cor_co_truncate,

    .bdrv_co_preadv                     = cor_co_preadv,
    .bdrv_co_pwritev                    = cor_co_pwritev,
@@ -25,7 +25,6 @@
 #include "qemu/osdep.h"
 #include "block/block_int.h"
 #include "qemu/job.h"
-#include "qemu/main-loop.h"
 #include "qapi/qapi-commands-block-core.h"
 #include "qapi/qapi-visit-block-core.h"
 #include "qapi/clone-visitor.h"
@@ -74,7 +74,6 @@ static ssize_t block_crypto_read_func(QCryptoBlock *block,
 struct BlockCryptoCreateData {
    BlockBackend *blk;
    uint64_t size;
-    PreallocMode prealloc;
 };


@@ -113,8 +112,8 @@ static ssize_t block_crypto_init_func(QCryptoBlock *block,
     * available to the guest, so we must take account of that
     * which will be used by the crypto header
     */
-    return blk_truncate(data->blk, data->size + headerlen, false,
-                        data->prealloc, errp);
+    return blk_truncate(data->blk, data->size + headerlen, PREALLOC_MODE_OFF,
+                        errp);
 }


@@ -252,7 +251,6 @@ static int block_crypto_open_generic(QCryptoBlockFormat format,
 static int block_crypto_co_create_generic(BlockDriverState *bs,
                                          int64_t size,
                                          QCryptoBlockCreateOptions *opts,
-                                          PreallocMode prealloc,
                                          Error **errp)
 {
    int ret;
@@ -268,14 +266,9 @@ static int block_crypto_co_create_generic(BlockDriverState *bs,
        goto cleanup;
    }

-    if (prealloc == PREALLOC_MODE_METADATA) {
-        prealloc = PREALLOC_MODE_OFF;
-    }
-
    data = (struct BlockCryptoCreateData) {
        .blk = blk,
        .size = size,
-        .prealloc = prealloc,
    };

    crypto = qcrypto_block_create(opts, NULL,
@@ -297,7 +290,7 @@ static int block_crypto_co_create_generic(BlockDriverState *bs,
 }

 static int coroutine_fn
-block_crypto_co_truncate(BlockDriverState *bs, int64_t offset, bool exact,
+block_crypto_co_truncate(BlockDriverState *bs, int64_t offset,
                         PreallocMode prealloc, Error **errp)
 {
    BlockCrypto *crypto = bs->opaque;
@@ -311,7 +304,7 @@ block_crypto_co_truncate(BlockDriverState *bs, int64_t offset, bool exact,

    offset += payload_offset;

-    return bdrv_co_truncate(bs->file, offset, exact, prealloc, errp);
+    return bdrv_co_truncate(bs->file, offset, prealloc, errp);
 }

 static void block_crypto_close(BlockDriverState *bs)
@@ -507,7 +500,6 @@ block_crypto_co_create_luks(BlockdevCreateOptions *create_options, Error **errp)
    BlockdevCreateOptionsLUKS *luks_opts;
    BlockDriverState *bs = NULL;
    QCryptoBlockCreateOptions create_opts;
-    PreallocMode preallocation = PREALLOC_MODE_OFF;
    int ret;

    assert(create_options->driver == BLOCKDEV_DRIVER_LUKS);
@@ -523,12 +515,8 @@ block_crypto_co_create_luks(BlockdevCreateOptions *create_options, Error **errp)
        .u.luks = *qapi_BlockdevCreateOptionsLUKS_base(luks_opts),
    };

-    if (luks_opts->has_preallocation) {
-        preallocation = luks_opts->preallocation;
-    }
-
    ret = block_crypto_co_create_generic(bs, luks_opts->size, &create_opts,
-                                         preallocation, errp);
+                                         errp);
    if (ret < 0) {
        goto fail;
    }
@@ -546,24 +534,12 @@ static int coroutine_fn block_crypto_co_create_opts_luks(const char *filename,
    QCryptoBlockCreateOptions *create_opts = NULL;
    BlockDriverState *bs = NULL;
    QDict *cryptoopts;
-    PreallocMode prealloc;
-    char *buf = NULL;
    int64_t size;
    int ret;
-    Error *local_err = NULL;

    /* Parse options */
    size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);

-    buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
-    prealloc = qapi_enum_parse(&PreallocMode_lookup, buf,
-                               PREALLOC_MODE_OFF, &local_err);
-    g_free(buf);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return -EINVAL;
-    }
-
    cryptoopts = qemu_opts_to_qdict_filtered(opts, NULL,
                                             &block_crypto_create_opts_luks,
                                             true);
@@ -589,7 +565,7 @@ static int coroutine_fn block_crypto_co_create_opts_luks(const char *filename,
    }

    /* Create format layer */
-    ret = block_crypto_co_create_generic(bs, size, create_opts, prealloc, errp);
+    ret = block_crypto_co_create_generic(bs, size, create_opts, errp);
    if (ret < 0) {
        goto fail;
    }
@@ -882,13 +882,7 @@ static void curl_setup_preadv(BlockDriverState *bs, CURLAIOCB *acb)
    trace_curl_setup_preadv(acb->bytes, start, state->range);
    curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);

-    if (curl_multi_add_handle(s->multi, state->curl) != CURLM_OK) {
-        state->acb[0] = NULL;
-        acb->ret = -EIO;
-
-        curl_clean_state(state);
-        goto out;
-    }
+    curl_multi_add_handle(s->multi, state->curl);

    /* Tell curl it needs to kick things off */
    curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0, &running);
@@ -26,11 +26,11 @@
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
-#include "qemu/main-loop.h"

 struct BdrvDirtyBitmap {
-    BlockDriverState *bs;
+    QemuMutex *mutex;
    HBitmap *bitmap;            /* Dirty bitmap implementation */
+    HBitmap *meta;              /* Meta dirty bitmap */
    bool busy;                  /* Bitmap is busy, it can't be used via QMP */
    BdrvDirtyBitmap *successor; /* Anonymous child, if any. */
    char *name;                 /* Optional non-empty unique ID */
@@ -48,9 +48,10 @@ struct BdrvDirtyBitmap {
    bool inconsistent;          /* bitmap is persistent, but inconsistent.
                                   It cannot be used at all in any way, except
                                   a QMP user can remove it. */
-    bool skip_store;            /* We are either migrating or deleting this
-                                 * bitmap; it should not be stored on the next
-                                 * inactivation. */
+    bool migration;             /* Bitmap is selected for migration, it should
+                                   not be stored on the next inactivation
+                                   (persistent flag doesn't matter until next
+                                   invalidation).*/
    QLIST_ENTRY(BdrvDirtyBitmap) list;
 };

@@ -71,12 +72,12 @@ static inline void bdrv_dirty_bitmaps_unlock(BlockDriverState *bs)

 void bdrv_dirty_bitmap_lock(BdrvDirtyBitmap *bitmap)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    qemu_mutex_lock(bitmap->mutex);
 }

 void bdrv_dirty_bitmap_unlock(BdrvDirtyBitmap *bitmap)
 {
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    qemu_mutex_unlock(bitmap->mutex);
 }

 /* Called with BQL or dirty_bitmap lock taken.  */
@@ -115,7 +116,7 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
        return NULL;
    }
    bitmap = g_new0(BdrvDirtyBitmap, 1);
-    bitmap->bs = bs;
+    bitmap->mutex = &bs->dirty_bitmap_mutex;
    bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(granularity));
    bitmap->size = bitmap_size;
    bitmap->name = g_strdup(name);
@@ -126,6 +127,36 @@ BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
    return bitmap;
 }

+/* bdrv_create_meta_dirty_bitmap
+ *
+ * Create a meta dirty bitmap that tracks the changes of bits in @bitmap. I.e.
+ * when a dirty status bit in @bitmap is changed (either from reset to set or
+ * the other way around), its respective meta dirty bitmap bit will be marked
+ * dirty as well.
+ *
+ * @bitmap: the block dirty bitmap for which to create a meta dirty bitmap.
+ * @chunk_size: how many bytes of bitmap data does each bit in the meta bitmap
+ * track.
+ */
+void bdrv_create_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                                   int chunk_size)
+{
+    assert(!bitmap->meta);
+    qemu_mutex_lock(bitmap->mutex);
+    bitmap->meta = hbitmap_create_meta(bitmap->bitmap,
+                                       chunk_size * BITS_PER_BYTE);
+    qemu_mutex_unlock(bitmap->mutex);
+}
+
+void bdrv_release_meta_dirty_bitmap(BdrvDirtyBitmap *bitmap)
+{
+    assert(bitmap->meta);
+    qemu_mutex_lock(bitmap->mutex);
+    hbitmap_free_meta(bitmap->bitmap);
+    bitmap->meta = NULL;
+    qemu_mutex_unlock(bitmap->mutex);
+}
+
 int64_t bdrv_dirty_bitmap_size(const BdrvDirtyBitmap *bitmap)
 {
    return bitmap->size;
@@ -149,9 +180,9 @@ static bool bdrv_dirty_bitmap_busy(const BdrvDirtyBitmap *bitmap)

 void bdrv_dirty_bitmap_set_busy(BdrvDirtyBitmap *bitmap, bool busy)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    qemu_mutex_lock(bitmap->mutex);
    bitmap->busy = busy;
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    qemu_mutex_unlock(bitmap->mutex);
 }

 /* Called with BQL taken.  */
@@ -237,7 +268,8 @@ int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, uint32_t flags,
 * The successor will be enabled if the parent bitmap was.
 * Called with BQL taken.
 */
-int bdrv_dirty_bitmap_create_successor(BdrvDirtyBitmap *bitmap, Error **errp)
+int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
+                                       BdrvDirtyBitmap *bitmap, Error **errp)
 {
    uint64_t granularity;
    BdrvDirtyBitmap *child;
@@ -253,7 +285,7 @@ int bdrv_dirty_bitmap_create_successor(BdrvDirtyBitmap *bitmap, Error **errp)

    /* Create an anonymous successor */
    granularity = bdrv_dirty_bitmap_granularity(bitmap);
-    child = bdrv_create_dirty_bitmap(bitmap->bs, granularity, NULL, errp);
+    child = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
    if (!child) {
        return -1;
    }
@@ -276,10 +308,10 @@ void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)
 /* Called with BQL taken. */
 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap)
 {
-    assert(bitmap->bs == bitmap->successor->bs);
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    assert(bitmap->mutex == bitmap->successor->mutex);
+    qemu_mutex_lock(bitmap->mutex);
    bdrv_enable_dirty_bitmap_locked(bitmap->successor);
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    qemu_mutex_unlock(bitmap->mutex);
 }

 /* Called within bdrv_dirty_bitmap_lock..unlock and with BQL taken.  */
@@ -288,6 +320,7 @@ static void bdrv_release_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)
    assert(!bitmap->active_iterators);
    assert(!bdrv_dirty_bitmap_busy(bitmap));
    assert(!bdrv_dirty_bitmap_has_successor(bitmap));
+    assert(!bitmap->meta);
    QLIST_REMOVE(bitmap, list);
    hbitmap_free(bitmap->bitmap);
    g_free(bitmap->name);
@@ -299,7 +332,8 @@ static void bdrv_release_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)
 * delete the old bitmap, and return a handle to the new bitmap.
 * Called with BQL taken.
 */
-BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BdrvDirtyBitmap *bitmap,
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
+                                            BdrvDirtyBitmap *bitmap,
                                            Error **errp)
 {
    char *name;
@@ -318,7 +352,7 @@ BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BdrvDirtyBitmap *bitmap,
    successor->persistent = bitmap->persistent;
    bitmap->persistent = false;
    bitmap->busy = false;
-    bdrv_release_dirty_bitmap(bitmap);
+    bdrv_release_dirty_bitmap(bs, bitmap);

    return successor;
 }
@@ -330,7 +364,8 @@ BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BdrvDirtyBitmap *bitmap,
 * The marged parent will be enabled if and only if the successor was enabled.
 * Called within bdrv_dirty_bitmap_lock..unlock and with BQL taken.
 */
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BdrvDirtyBitmap *parent,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BlockDriverState *bs,
+                                                  BdrvDirtyBitmap *parent,
                                                  Error **errp)
 {
    BdrvDirtyBitmap *successor = parent->successor;
@@ -354,14 +389,15 @@ BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap_locked(BdrvDirtyBitmap *parent,
 }

 /* Called with BQL taken. */
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BdrvDirtyBitmap *parent,
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
+                                           BdrvDirtyBitmap *parent,
                                           Error **errp)
 {
    BdrvDirtyBitmap *ret;

-    bdrv_dirty_bitmaps_lock(parent->bs);
-    ret = bdrv_reclaim_dirty_bitmap_locked(parent, errp);
-    bdrv_dirty_bitmaps_unlock(parent->bs);
+    qemu_mutex_lock(parent->mutex);
+    ret = bdrv_reclaim_dirty_bitmap_locked(bs, parent, errp);
+    qemu_mutex_unlock(parent->mutex);

    return ret;
 }
@@ -386,10 +422,8 @@ void bdrv_dirty_bitmap_truncate(BlockDriverState *bs, int64_t bytes)
 }

 /* Called with BQL taken.  */
-void bdrv_release_dirty_bitmap(BdrvDirtyBitmap *bitmap)
+void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap)
 {
-    BlockDriverState *bs = bitmap->bs;
-
    bdrv_dirty_bitmaps_lock(bs);
    bdrv_release_dirty_bitmap_locked(bitmap);
    bdrv_dirty_bitmaps_unlock(bs);
@@ -422,135 +456,27 @@ void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs)
 * not fail.
 * This function doesn't release corresponding BdrvDirtyBitmap.
 */
-static int coroutine_fn
-bdrv_co_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
-                                       Error **errp)
+void bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs,
+                                         const char *name,
+                                         Error **errp)
 {
-    if (bs->drv && bs->drv->bdrv_co_remove_persistent_dirty_bitmap) {
-        return bs->drv->bdrv_co_remove_persistent_dirty_bitmap(bs, name, errp);
-    }
-
-    return 0;
-}
-
-typedef struct BdrvRemovePersistentDirtyBitmapCo {
-    BlockDriverState *bs;
-    const char *name;
-    Error **errp;
-    int ret;
-} BdrvRemovePersistentDirtyBitmapCo;
-
-static void coroutine_fn
-bdrv_co_remove_persistent_dirty_bitmap_entry(void *opaque)
-{
-    BdrvRemovePersistentDirtyBitmapCo *s = opaque;
-
-    s->ret = bdrv_co_remove_persistent_dirty_bitmap(s->bs, s->name, s->errp);
-    aio_wait_kick();
-}
-
-int bdrv_remove_persistent_dirty_bitmap(BlockDriverState *bs, const char *name,
-                                        Error **errp)
-{
-    if (qemu_in_coroutine()) {
-        return bdrv_co_remove_persistent_dirty_bitmap(bs, name, errp);
-    } else {
-        Coroutine *co;
-        BdrvRemovePersistentDirtyBitmapCo s = {
-            .bs = bs,
-            .name = name,
-            .errp = errp,
-            .ret = -EINPROGRESS,
-        };
-
-        co = qemu_coroutine_create(bdrv_co_remove_persistent_dirty_bitmap_entry,
-                                   &s);
-        bdrv_coroutine_enter(bs, co);
-        BDRV_POLL_WHILE(bs, s.ret == -EINPROGRESS);
-
-        return s.ret;
-    }
-}
-
-static bool coroutine_fn
-bdrv_co_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
-                                   uint32_t granularity, Error **errp)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (!drv) {
-        error_setg_errno(errp, ENOMEDIUM,
-                         "Can't store persistent bitmaps to %s",
-                         bdrv_get_device_or_node_name(bs));
-        return false;
-    }
-
-    if (!drv->bdrv_co_can_store_new_dirty_bitmap) {
-        error_setg_errno(errp, ENOTSUP,
-                         "Can't store persistent bitmaps to %s",
-                         bdrv_get_device_or_node_name(bs));
-        return false;
-    }
-
-    return drv->bdrv_co_can_store_new_dirty_bitmap(bs, name, granularity, errp);
-}
-
-typedef struct BdrvCanStoreNewDirtyBitmapCo {
-    BlockDriverState *bs;
-    const char *name;
-    uint32_t granularity;
-    Error **errp;
-    bool ret;
-
-    bool in_progress;
-} BdrvCanStoreNewDirtyBitmapCo;
-
-static void coroutine_fn bdrv_co_can_store_new_dirty_bitmap_entry(void *opaque)
-{
-    BdrvCanStoreNewDirtyBitmapCo *s = opaque;
-
-    s->ret = bdrv_co_can_store_new_dirty_bitmap(s->bs, s->name, s->granularity,
-                                                s->errp);
-    s->in_progress = false;
-    aio_wait_kick();
-}
-
-bool bdrv_can_store_new_dirty_bitmap(BlockDriverState *bs, const char *name,
-                                     uint32_t granularity, Error **errp)
-{
-    if (qemu_in_coroutine()) {
-        return bdrv_co_can_store_new_dirty_bitmap(bs, name, granularity, errp);
-    } else {
-        Coroutine *co;
-        BdrvCanStoreNewDirtyBitmapCo s = {
-            .bs = bs,
-            .name = name,
-            .granularity = granularity,
-            .errp = errp,
-            .in_progress = true,
-        };
-
-        co = qemu_coroutine_create(bdrv_co_can_store_new_dirty_bitmap_entry,
-                                   &s);
-        bdrv_coroutine_enter(bs, co);
-        BDRV_POLL_WHILE(bs, s.in_progress);
-
-        return s.ret;
+    if (bs->drv && bs->drv->bdrv_remove_persistent_dirty_bitmap) {
+        bs->drv->bdrv_remove_persistent_dirty_bitmap(bs, name, errp);
    }
 }

 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    bdrv_dirty_bitmap_lock(bitmap);
    bitmap->disabled = true;
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    bdrv_dirty_bitmap_unlock(bitmap);
 }

 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    bdrv_dirty_bitmap_lock(bitmap);
    bdrv_enable_dirty_bitmap_locked(bitmap);
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    bdrv_dirty_bitmap_unlock(bitmap);
 }

 BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
@@ -583,19 +509,14 @@ BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
 }

 /* Called within bdrv_dirty_bitmap_lock..unlock */
-bool bdrv_dirty_bitmap_get_locked(BdrvDirtyBitmap *bitmap, int64_t offset)
+bool bdrv_get_dirty_locked(BlockDriverState *bs, BdrvDirtyBitmap *bitmap,
+                           int64_t offset)
 {
-    return hbitmap_get(bitmap->bitmap, offset);
-}
-
-bool bdrv_dirty_bitmap_get(BdrvDirtyBitmap *bitmap, int64_t offset)
-{
-    bool ret;
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
-    ret = bdrv_dirty_bitmap_get_locked(bitmap, offset);
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
-
-    return ret;
+    if (bitmap) {
+        return hbitmap_get(bitmap->bitmap, offset);
+    } else {
+        return false;
+    }
 }

 /**
@@ -632,6 +553,15 @@ BdrvDirtyBitmapIter *bdrv_dirty_iter_new(BdrvDirtyBitmap *bitmap)
    return iter;
 }

+BdrvDirtyBitmapIter *bdrv_dirty_meta_iter_new(BdrvDirtyBitmap *bitmap)
+{
+    BdrvDirtyBitmapIter *iter = g_new(BdrvDirtyBitmapIter, 1);
+    hbitmap_iter_init(&iter->hbi, bitmap->meta, 0);
+    iter->bitmap = bitmap;
+    bitmap->active_iterators++;
+    return iter;
+}
+
 void bdrv_dirty_iter_free(BdrvDirtyBitmapIter *iter)
 {
    if (!iter) {
@@ -658,9 +588,9 @@ void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
                           int64_t offset, int64_t bytes)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    bdrv_dirty_bitmap_lock(bitmap);
    bdrv_set_dirty_bitmap_locked(bitmap, offset, bytes);
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    bdrv_dirty_bitmap_unlock(bitmap);
 }

 /* Called within bdrv_dirty_bitmap_lock..unlock */
@@ -674,15 +604,15 @@ void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,
 void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
                             int64_t offset, int64_t bytes)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    bdrv_dirty_bitmap_lock(bitmap);
    bdrv_reset_dirty_bitmap_locked(bitmap, offset, bytes);
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    bdrv_dirty_bitmap_unlock(bitmap);
 }

 void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out)
 {
    assert(!bdrv_dirty_bitmap_readonly(bitmap));
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    bdrv_dirty_bitmap_lock(bitmap);
    if (!out) {
        hbitmap_reset_all(bitmap->bitmap);
    } else {
@@ -691,7 +621,7 @@ void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out)
                                       hbitmap_granularity(backup));
        *out = backup;
    }
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    bdrv_dirty_bitmap_unlock(bitmap);
 }

 void bdrv_restore_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *backup)
@@ -778,6 +708,11 @@ int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)
    return hbitmap_count(bitmap->bitmap);
 }

+int64_t bdrv_get_meta_dirty_count(BdrvDirtyBitmap *bitmap)
+{
+    return hbitmap_count(bitmap->meta);
+}
+
 bool bdrv_dirty_bitmap_readonly(const BdrvDirtyBitmap *bitmap)
 {
    return bitmap->readonly;
@@ -786,9 +721,9 @@ bool bdrv_dirty_bitmap_readonly(const BdrvDirtyBitmap *bitmap)
 /* Called with BQL taken. */
 void bdrv_dirty_bitmap_set_readonly(BdrvDirtyBitmap *bitmap, bool value)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    qemu_mutex_lock(bitmap->mutex);
    bitmap->readonly = value;
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    qemu_mutex_unlock(bitmap->mutex);
 }

 bool bdrv_has_readonly_bitmaps(BlockDriverState *bs)
@@ -806,32 +741,32 @@ bool bdrv_has_readonly_bitmaps(BlockDriverState *bs)
 /* Called with BQL taken. */
 void bdrv_dirty_bitmap_set_persistence(BdrvDirtyBitmap *bitmap, bool persistent)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    qemu_mutex_lock(bitmap->mutex);
    bitmap->persistent = persistent;
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    qemu_mutex_unlock(bitmap->mutex);
 }

 /* Called with BQL taken. */
 void bdrv_dirty_bitmap_set_inconsistent(BdrvDirtyBitmap *bitmap)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
+    qemu_mutex_lock(bitmap->mutex);
    assert(bitmap->persistent == true);
    bitmap->inconsistent = true;
    bitmap->disabled = true;
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    qemu_mutex_unlock(bitmap->mutex);
 }

 /* Called with BQL taken. */
-void bdrv_dirty_bitmap_skip_store(BdrvDirtyBitmap *bitmap, bool skip)
+void bdrv_dirty_bitmap_set_migration(BdrvDirtyBitmap *bitmap, bool migration)
 {
-    bdrv_dirty_bitmaps_lock(bitmap->bs);
-    bitmap->skip_store = skip;
-    bdrv_dirty_bitmaps_unlock(bitmap->bs);
+    qemu_mutex_lock(bitmap->mutex);
+    bitmap->migration = migration;
+    qemu_mutex_unlock(bitmap->mutex);
 }

 bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap)
 {
-    return bitmap->persistent && !bitmap->skip_store;
+    return bitmap->persistent && !bitmap->migration;
 }

 bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap)
@@ -839,14 +774,23 @@ bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap)
    return bitmap->inconsistent;
 }

-BdrvDirtyBitmap *bdrv_dirty_bitmap_first(BlockDriverState *bs)
+bool bdrv_has_changed_persistent_bitmaps(BlockDriverState *bs)
 {
-    return QLIST_FIRST(&bs->dirty_bitmaps);
+    BdrvDirtyBitmap *bm;
+    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
+        if (bm->persistent && !bm->readonly && !bm->migration) {
+            return true;
+        }
+    }
+
+    return false;
 }

-BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap)
+BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BlockDriverState *bs,
+                                        BdrvDirtyBitmap *bitmap)
 {
-    return QLIST_NEXT(bitmap, list);
+    return bitmap == NULL ? QLIST_FIRST(&bs->dirty_bitmaps) :
+                            QLIST_NEXT(bitmap, list);
 }

 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp)
@@ -866,20 +810,14 @@ bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,
    return hbitmap_next_dirty_area(bitmap->bitmap, offset, bytes);
 }

-/**
- * bdrv_merge_dirty_bitmap: merge src into dest.
- * Ensures permissions on bitmaps are reasonable; use for public API.
- *
- * @backup: If provided, make a copy of dest here prior to merge.
- */
 void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src,
                             HBitmap **backup, Error **errp)
 {
    bool ret;

-    bdrv_dirty_bitmaps_lock(dest->bs);
-    if (src->bs != dest->bs) {
-        bdrv_dirty_bitmaps_lock(src->bs);
+    qemu_mutex_lock(dest->mutex);
+    if (src->mutex != dest->mutex) {
+        qemu_mutex_lock(src->mutex);
    }

    if (bdrv_dirty_bitmap_check(dest, BDRV_BITMAP_DEFAULT, errp)) {
@@ -895,42 +833,6 @@ void bdrv_merge_dirty_bitmap(BdrvDirtyBitmap *dest, const BdrvDirtyBitmap *src,
        goto out;
    }

-    ret = bdrv_dirty_bitmap_merge_internal(dest, src, backup, false);
-    assert(ret);
-
-out:
-    bdrv_dirty_bitmaps_unlock(dest->bs);
-    if (src->bs != dest->bs) {
-        bdrv_dirty_bitmaps_unlock(src->bs);
-    }
-}
-
-/**
- * bdrv_dirty_bitmap_merge_internal: merge src into dest.
- * Does NOT check bitmap permissions; not suitable for use as public API.
- *
- * @backup: If provided, make a copy of dest here prior to merge.
- * @lock: If true, lock and unlock bitmaps on the way in/out.
- * returns true if the merge succeeded; false if unattempted.
- */
-bool bdrv_dirty_bitmap_merge_internal(BdrvDirtyBitmap *dest,
-                                      const BdrvDirtyBitmap *src,
-                                      HBitmap **backup,
-                                      bool lock)
-{
-    bool ret;
-
-    assert(!bdrv_dirty_bitmap_readonly(dest));
-    assert(!bdrv_dirty_bitmap_inconsistent(dest));
-    assert(!bdrv_dirty_bitmap_inconsistent(src));
-
-    if (lock) {
-        bdrv_dirty_bitmaps_lock(dest->bs);
-        if (src->bs != dest->bs) {
-            bdrv_dirty_bitmaps_lock(src->bs);
-        }
-    }
-
    if (backup) {
        *backup = dest->bitmap;
        dest->bitmap = hbitmap_alloc(dest->size, hbitmap_granularity(*backup));
@@ -938,13 +840,11 @@ bool bdrv_dirty_bitmap_merge_internal(BdrvDirtyBitmap *dest,
    } else {
        ret = hbitmap_merge(dest->bitmap, src->bitmap, dest->bitmap);
    }
+    assert(ret);

-    if (lock) {
-        bdrv_dirty_bitmaps_unlock(dest->bs);
-        if (src->bs != dest->bs) {
-            bdrv_dirty_bitmaps_unlock(src->bs);
-        }
+out:
+    qemu_mutex_unlock(dest->mutex);
+    if (src->mutex != dest->mutex) {
+        qemu_mutex_unlock(src->mutex);
    }
-
-    return ret;
 }
@@ -697,8 +697,8 @@ dmg_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
    int nb_sectors = bytes >> BDRV_SECTOR_BITS;
    int ret, i;

-    assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
-    assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);

    qemu_co_mutex_lock(&s->lock);

@@ -161,11 +161,6 @@ typedef struct BDRVRawState {
    bool needs_alignment;
    bool drop_cache;
    bool check_cache_dropped;
-    struct {
-        uint64_t discard_nb_ok;
-        uint64_t discard_nb_failed;
-        uint64_t discard_bytes_ok;
-    } stats;

    PRManager *pr_mgr;
 } BDRVRawState;
@@ -222,7 +217,7 @@ static int raw_normalize_devicepath(const char **filename, Error **errp)
    fname = *filename;
    dp = strrchr(fname, '/');
    if (lstat(fname, &sb) < 0) {
-        error_setg_file_open(errp, errno, fname);
+        error_setg_errno(errp, errno, "%s: stat failed", fname);
        return -errno;
    }

@@ -327,7 +322,7 @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
 {
    BDRVRawState *s = bs->opaque;
    char *buf;
-    size_t max_align = MAX(MAX_BLOCKSIZE, qemu_real_host_page_size);
+    size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());
    size_t alignments[] = {1, 512, 1024, 2048, 4096};

    /* For SCSI generic devices the alignment is not really used.
@@ -385,7 +380,7 @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
        for (i = 0; i < ARRAY_SIZE(alignments); i++) {
            align = alignments[i];
            if (raw_is_io_aligned(fd, buf + align, max_align)) {
-                /* Fallback to request_alignment. */
+                /* Fallback to request_aligment. */
                s->buf_align = (align != 1) ? align : bs->bl.request_alignment;
                break;
            }
@@ -566,7 +561,7 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
    ret = fd < 0 ? -errno : 0;

    if (ret < 0) {
-        error_setg_file_open(errp, -ret, filename);
+        error_setg_errno(errp, -ret, "Could not open '%s'", filename);
        if (ret == -EROFS) {
            ret = -EACCES;
        }
@@ -1136,14 +1131,13 @@ static void raw_refresh_limits(BlockDriverState *bs, Error **errp)

        ret = sg_get_max_segments(s->fd);
        if (ret > 0) {
-            bs->bl.max_transfer = MIN(bs->bl.max_transfer,
-                                      ret * qemu_real_host_page_size);
+            bs->bl.max_transfer = MIN(bs->bl.max_transfer, ret * getpagesize());
        }
    }

    raw_probe_alignment(bs, s->fd, errp);
    bs->bl.min_mem_alignment = s->buf_align;
-    bs->bl.opt_mem_alignment = MAX(s->buf_align, qemu_real_host_page_size);
+    bs->bl.opt_mem_alignment = MAX(s->buf_align, getpagesize());
 }

 static int check_for_dasd(int fd)
@@ -1508,12 +1502,12 @@ static ssize_t handle_aiocb_write_zeroes_block(RawPosixAIOData *aiocb)
        } while (errno == EINTR);

        ret = translate_err(-errno);
-        if (ret == -ENOTSUP) {
-            s->has_write_zeroes = false;
-        }
    }
 #endif

+    if (ret == -ENOTSUP) {
+        s->has_write_zeroes = false;
+    }
    return ret;
 }

@@ -1533,13 +1527,6 @@ static int handle_aiocb_write_zeroes(void *opaque)
    if (s->has_write_zeroes) {
        int ret = do_fallocate(s->fd, FALLOC_FL_ZERO_RANGE,
                               aiocb->aio_offset, aiocb->aio_nbytes);
-        if (ret == -EINVAL) {
-            /*
-             * Allow falling back to pwrite for file systems that
-             * do not support fallocate() for an unaligned byte range.
-             */
-            return -ENOTSUP;
-        }
        if (ret == 0 || ret != -ENOTSUP) {
            return ret;
        }
@@ -1706,7 +1693,7 @@ static int allocate_first_block(int fd, size_t max_size)
    size_t write_size = (max_size < MAX_BLOCKSIZE)
        ? BDRV_SECTOR_SIZE
        : MAX_BLOCKSIZE;
-    size_t max_align = MAX(MAX_BLOCKSIZE, qemu_real_host_page_size);
+    size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());
    void *buf;
    ssize_t n;
    int ret;
@@ -2020,8 +2007,7 @@ raw_regular_truncate(BlockDriverState *bs, int fd, int64_t offset,
 }

 static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
-                                        bool exact, PreallocMode prealloc,
-                                        Error **errp)
+                                        PreallocMode prealloc, Error **errp)
 {
    BDRVRawState *s = bs->opaque;
    struct stat st;
@@ -2034,7 +2020,6 @@ static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
    }

    if (S_ISREG(st.st_mode)) {
-        /* Always resizes to the exact @offset */
        return raw_regular_truncate(bs, s->fd, offset, prealloc, errp);
    }

@@ -2045,12 +2030,7 @@ static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
    }

    if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode)) {
-        int64_t cur_length = raw_getlength(bs);
-
-        if (offset != cur_length && exact) {
-            error_setg(errp, "Cannot resize device files");
-            return -ENOTSUP;
-        } else if (offset > cur_length) {
+        if (offset > raw_getlength(bs)) {
            error_setg(errp, "Cannot grow device files");
            return -EINVAL;
        }
@@ -2673,22 +2653,11 @@ static void coroutine_fn raw_co_invalidate_cache(BlockDriverState *bs,
 #endif /* !__linux__ */
 }

-static void raw_account_discard(BDRVRawState *s, uint64_t nbytes, int ret)
-{
-    if (ret) {
-        s->stats.discard_nb_failed++;
-    } else {
-        s->stats.discard_nb_ok++;
-        s->stats.discard_bytes_ok += nbytes;
-    }
-}
-
 static coroutine_fn int
 raw_do_pdiscard(BlockDriverState *bs, int64_t offset, int bytes, bool blkdev)
 {
    BDRVRawState *s = bs->opaque;
    RawPosixAIOData acb;
-    int ret;

    acb = (RawPosixAIOData) {
        .bs             = bs,
@@ -2702,9 +2671,7 @@ raw_do_pdiscard(BlockDriverState *bs, int64_t offset, int bytes, bool blkdev)
        acb.aio_type |= QEMU_AIO_BLKDEV;
    }

-    ret = raw_thread_pool_submit(bs, handle_aiocb_discard, &acb);
-    raw_account_discard(s, bytes, ret);
-    return ret;
+    return raw_thread_pool_submit(bs, handle_aiocb_discard, &acb);
 }

 static coroutine_fn int
@@ -2797,36 +2764,6 @@ static int raw_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
    return 0;
 }

-static BlockStatsSpecificFile get_blockstats_specific_file(BlockDriverState *bs)
-{
-    BDRVRawState *s = bs->opaque;
-    return (BlockStatsSpecificFile) {
-        .discard_nb_ok = s->stats.discard_nb_ok,
-        .discard_nb_failed = s->stats.discard_nb_failed,
-        .discard_bytes_ok = s->stats.discard_bytes_ok,
-    };
-}
-
-static BlockStatsSpecific *raw_get_specific_stats(BlockDriverState *bs)
-{
-    BlockStatsSpecific *stats = g_new(BlockStatsSpecific, 1);
-
-    stats->driver = BLOCKDEV_DRIVER_FILE;
-    stats->u.file = get_blockstats_specific_file(bs);
-
-    return stats;
-}
-
-static BlockStatsSpecific *hdev_get_specific_stats(BlockDriverState *bs)
-{
-    BlockStatsSpecific *stats = g_new(BlockStatsSpecific, 1);
-
-    stats->driver = BLOCKDEV_DRIVER_HOST_DEVICE;
-    stats->u.host_device = get_blockstats_specific_file(bs);
-
-    return stats;
-}
-
 static QemuOptsList raw_create_opts = {
    .name = "raw-create-opts",
    .head = QTAILQ_HEAD_INITIALIZER(raw_create_opts.head),
@@ -3013,7 +2950,6 @@ BlockDriver bdrv_file = {
    .bdrv_co_create = raw_co_create,
    .bdrv_co_create_opts = raw_co_create_opts,
    .bdrv_has_zero_init = bdrv_has_zero_init_1,
-    .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1,
    .bdrv_co_block_status = raw_co_block_status,
    .bdrv_co_invalidate_cache = raw_co_invalidate_cache,
    .bdrv_co_pwrite_zeroes = raw_co_pwrite_zeroes,
@@ -3034,7 +2970,6 @@ BlockDriver bdrv_file = {
    .bdrv_get_info = raw_get_info,
    .bdrv_get_allocated_file_size
                        = raw_get_allocated_file_size,
-    .bdrv_get_specific_stats = raw_get_specific_stats,
    .bdrv_check_perm = raw_check_perm,
    .bdrv_set_perm   = raw_set_perm,
    .bdrv_abort_perm_update = raw_abort_perm_update,
@@ -3394,12 +3329,10 @@ static int fd_open(BlockDriverState *bs)
 static coroutine_fn int
 hdev_co_pdiscard(BlockDriverState *bs, int64_t offset, int bytes)
 {
-    BDRVRawState *s = bs->opaque;
    int ret;

    ret = fd_open(bs);
    if (ret < 0) {
-        raw_account_discard(s, bytes, ret);
        return ret;
    }
    return raw_do_pdiscard(bs, offset, bytes, true);
@@ -3513,7 +3446,6 @@ static BlockDriver bdrv_host_device = {
    .bdrv_get_info = raw_get_info,
    .bdrv_get_allocated_file_size
                        = raw_get_allocated_file_size,
-    .bdrv_get_specific_stats = hdev_get_specific_stats,
    .bdrv_check_perm = raw_check_perm,
    .bdrv_set_perm   = raw_set_perm,
    .bdrv_abort_perm_update = raw_abort_perm_update,
@@ -468,8 +468,7 @@ static void raw_close(BlockDriverState *bs)
 }

 static int coroutine_fn raw_co_truncate(BlockDriverState *bs, int64_t offset,
-                                        bool exact, PreallocMode prealloc,
-                                        Error **errp)
+                                        PreallocMode prealloc, Error **errp)
 {
    BDRVRawState *s = bs->opaque;
    LONG low, high;
@@ -636,7 +635,6 @@ BlockDriver bdrv_file = {
    .bdrv_close         = raw_close,
    .bdrv_co_create_opts = raw_co_create_opts,
    .bdrv_has_zero_init = bdrv_has_zero_init_1,
-    .bdrv_has_zero_init_truncate = bdrv_has_zero_init_1,

    .bdrv_aio_preadv    = raw_aio_preadv,
    .bdrv_aio_pwritev   = raw_aio_pwritev,
@@ -1225,7 +1225,6 @@ static coroutine_fn int qemu_gluster_co_rw(BlockDriverState *bs,

 static coroutine_fn int qemu_gluster_co_truncate(BlockDriverState *bs,
                                                 int64_t offset,
-                                                 bool exact,
                                                 PreallocMode prealloc,
                                                 Error **errp)
 {
@@ -1568,7 +1567,6 @@ static BlockDriver bdrv_gluster = {
    .bdrv_co_writev               = qemu_gluster_co_writev,
    .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
    .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
-    .bdrv_has_zero_init_truncate  = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
@@ -1600,7 +1598,6 @@ static BlockDriver bdrv_gluster_tcp = {
    .bdrv_co_writev               = qemu_gluster_co_writev,
    .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
    .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
-    .bdrv_has_zero_init_truncate  = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
@@ -1632,7 +1629,6 @@ static BlockDriver bdrv_gluster_unix = {
    .bdrv_co_writev               = qemu_gluster_co_writev,
    .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
    .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
-    .bdrv_has_zero_init_truncate  = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
@@ -1670,7 +1666,6 @@ static BlockDriver bdrv_gluster_rdma = {
    .bdrv_co_writev               = qemu_gluster_co_writev,
    .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
    .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
-    .bdrv_has_zero_init_truncate  = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
@@ -32,8 +32,6 @@
 #include "qemu/cutils.h"
 #include "qapi/error.h"
 #include "qemu/error-report.h"
-#include "qemu/main-loop.h"
-#include "sysemu/replay.h"

 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */

@@ -147,8 +145,7 @@ void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)

    /* Default alignment based on whether driver has byte interface */
    bs->bl.request_alignment = (drv->bdrv_co_preadv ||
-                                drv->bdrv_aio_preadv ||
-                                drv->bdrv_co_preadv_part) ? 1 : 512;
+                                drv->bdrv_aio_preadv) ? 1 : 512;

    /* Take some limits from the children as a default */
    if (bs->file) {
@@ -160,7 +157,7 @@ void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
        bdrv_merge_limits(&bs->bl, &bs->file->bs->bl);
    } else {
        bs->bl.min_mem_alignment = 512;
-        bs->bl.opt_mem_alignment = qemu_real_host_page_size;
+        bs->bl.opt_mem_alignment = getpagesize();

        /* Safe default since most protocols use readv()/writev()/etc */
        bs->bl.max_iov = IOV_MAX;
@@ -369,8 +366,8 @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
    if (bs) {
        bdrv_inc_in_flight(bs);
    }
-    replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs),
-                                     bdrv_co_drain_bh_cb, &data);
+    aio_bh_schedule_oneshot(bdrv_get_aio_context(bs),
+                            bdrv_co_drain_bh_cb, &data);

    qemu_coroutine_yield();
    /* If we are resumed from some other event (such as an aio completion or a
@@ -601,15 +598,6 @@ void bdrv_drain_all_begin(void)
        return;
    }

-    /*
-     * bdrv queue is managed by record/replay,
-     * waiting for finishing the I/O requests may
-     * be infinite
-     */
-    if (replay_events_enabled()) {
-        return;
-    }
-
    /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
     * loop AioContext, so make sure we're in the main context. */
    assert(qemu_get_current_aio_context() == qemu_get_aio_context());
@@ -639,15 +627,6 @@ void bdrv_drain_all_end(void)
    BlockDriverState *bs = NULL;
    int drained_end_counter = 0;

-    /*
-     * bdrv queue is managed by record/replay,
-     * waiting for finishing the I/O requests may
-     * be endless
-     */
-    if (replay_events_enabled()) {
-        return;
-    }
-
    while ((bs = bdrv_next_all_states(bs))) {
        AioContext *aio_context = bdrv_get_aio_context(bs);

@@ -1082,14 +1061,11 @@ static void bdrv_co_io_em_complete(void *opaque, int ret)

 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
                                           uint64_t offset, uint64_t bytes,
-                                           QEMUIOVector *qiov,
-                                           size_t qiov_offset, int flags)
+                                           QEMUIOVector *qiov, int flags)
 {
    BlockDriver *drv = bs->drv;
    int64_t sector_num;
    unsigned int nb_sectors;
-    QEMUIOVector local_qiov;
-    int ret;

    assert(!(flags & ~BDRV_REQ_MASK));
    assert(!(flags & BDRV_REQ_NO_FALLBACK));
@@ -1098,19 +1074,8 @@ static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
        return -ENOMEDIUM;
    }

-    if (drv->bdrv_co_preadv_part) {
-        return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
-                                        flags);
-    }
-
-    if (qiov_offset > 0 || bytes != qiov->size) {
-        qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
-        qiov = &local_qiov;
-    }
-
    if (drv->bdrv_co_preadv) {
-        ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
-        goto out;
+        return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
    }

    if (drv->bdrv_aio_preadv) {
@@ -1122,42 +1087,31 @@ static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
        acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
                                   bdrv_co_io_em_complete, &co);
        if (acb == NULL) {
-            ret = -EIO;
-            goto out;
+            return -EIO;
        } else {
            qemu_coroutine_yield();
-            ret = co.ret;
-            goto out;
+            return co.ret;
        }
    }

    sector_num = offset >> BDRV_SECTOR_BITS;
    nb_sectors = bytes >> BDRV_SECTOR_BITS;

-    assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
-    assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
    assert(bytes <= BDRV_REQUEST_MAX_BYTES);
    assert(drv->bdrv_co_readv);

-    ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
-
-out:
-    if (qiov == &local_qiov) {
-        qemu_iovec_destroy(&local_qiov);
-    }
-
-    return ret;
+    return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
 }

 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
                                            uint64_t offset, uint64_t bytes,
-                                            QEMUIOVector *qiov,
-                                            size_t qiov_offset, int flags)
+                                            QEMUIOVector *qiov, int flags)
 {
    BlockDriver *drv = bs->drv;
    int64_t sector_num;
    unsigned int nb_sectors;
-    QEMUIOVector local_qiov;
    int ret;

    assert(!(flags & ~BDRV_REQ_MASK));
@@ -1167,18 +1121,6 @@ static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
        return -ENOMEDIUM;
    }

-    if (drv->bdrv_co_pwritev_part) {
-        ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
-                                        flags & bs->supported_write_flags);
-        flags &= ~bs->supported_write_flags;
-        goto emulate_flags;
-    }
-
-    if (qiov_offset > 0 || bytes != qiov->size) {
-        qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
-        qiov = &local_qiov;
-    }
-
    if (drv->bdrv_co_pwritev) {
        ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
                                   flags & bs->supported_write_flags);
@@ -1208,8 +1150,8 @@ static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
    sector_num = offset >> BDRV_SECTOR_BITS;
    nb_sectors = bytes >> BDRV_SECTOR_BITS;

-    assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
-    assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
    assert(bytes <= BDRV_REQUEST_MAX_BYTES);

    assert(drv->bdrv_co_writev);
@@ -1222,49 +1164,28 @@ emulate_flags:
        ret = bdrv_co_flush(bs);
    }

-    if (qiov == &local_qiov) {
-        qemu_iovec_destroy(&local_qiov);
-    }
-
    return ret;
 }

 static int coroutine_fn
 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
-                               uint64_t bytes, QEMUIOVector *qiov,
-                               size_t qiov_offset)
+                               uint64_t bytes, QEMUIOVector *qiov)
 {
    BlockDriver *drv = bs->drv;
-    QEMUIOVector local_qiov;
-    int ret;

    if (!drv) {
        return -ENOMEDIUM;
    }

-    if (!block_driver_can_compress(drv)) {
+    if (!drv->bdrv_co_pwritev_compressed) {
        return -ENOTSUP;
    }

-    if (drv->bdrv_co_pwritev_compressed_part) {
-        return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
-                                                    qiov, qiov_offset);
-    }
-
-    if (qiov_offset == 0) {
-        return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
-    }
-
-    qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
-    ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
-    qemu_iovec_destroy(&local_qiov);
-
-    return ret;
+    return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
 }

 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
-        int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-        size_t qiov_offset, int flags)
+        int64_t offset, unsigned int bytes, QEMUIOVector *qiov)
 {
    BlockDriverState *bs = child->bs;

@@ -1273,9 +1194,10 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
     * modifying the image file.  This is critical for zero-copy guest I/O
     * where anything might happen inside guest memory.
     */
-    void *bounce_buffer = NULL;
+    void *bounce_buffer;

    BlockDriver *drv = bs->drv;
+    QEMUIOVector local_qiov;
    int64_t cluster_offset;
    int64_t cluster_bytes;
    size_t skip_bytes;
@@ -1283,18 +1205,11 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
    int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
                                    BDRV_REQUEST_MAX_BYTES);
    unsigned int progress = 0;
-    bool skip_write;

    if (!drv) {
        return -ENOMEDIUM;
    }

-    /*
-     * Do not write anything when the BDS is inactive.  That is not
-     * allowed, and it would not help.
-     */
-    skip_write = (bs->open_flags & BDRV_O_INACTIVE);
-
    /* FIXME We cannot require callers to have write permissions when all they
     * are doing is a read request. If we did things right, write permissions
     * would be obtained anyway, but internally by the copy-on-read code. As
@@ -1315,53 +1230,42 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
    trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
                                   cluster_offset, cluster_bytes);

+    bounce_buffer = qemu_try_blockalign(bs,
+                                        MIN(MIN(max_transfer, cluster_bytes),
+                                            MAX_BOUNCE_BUFFER));
+    if (bounce_buffer == NULL) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
    while (cluster_bytes) {
        int64_t pnum;

-        if (skip_write) {
-            ret = 1; /* "already allocated", so nothing will be copied */
+        ret = bdrv_is_allocated(bs, cluster_offset,
+                                MIN(cluster_bytes, max_transfer), &pnum);
+        if (ret < 0) {
+            /* Safe to treat errors in querying allocation as if
+             * unallocated; we'll probably fail again soon on the
+             * read, but at least that will set a decent errno.
+             */
            pnum = MIN(cluster_bytes, max_transfer);
-        } else {
-            ret = bdrv_is_allocated(bs, cluster_offset,
-                                    MIN(cluster_bytes, max_transfer), &pnum);
-            if (ret < 0) {
-                /*
-                 * Safe to treat errors in querying allocation as if
-                 * unallocated; we'll probably fail again soon on the
-                 * read, but at least that will set a decent errno.
-                 */
-                pnum = MIN(cluster_bytes, max_transfer);
-            }
-
-            /* Stop at EOF if the image ends in the middle of the cluster */
-            if (ret == 0 && pnum == 0) {
-                assert(progress >= bytes);
-                break;
-            }
-
-            assert(skip_bytes < pnum);
        }

-        if (ret <= 0) {
-            QEMUIOVector local_qiov;
+        /* Stop at EOF if the image ends in the middle of the cluster */
+        if (ret == 0 && pnum == 0) {
+            assert(progress >= bytes);
+            break;
+        }

+        assert(skip_bytes < pnum);
+
+        if (ret <= 0) {
            /* Must copy-on-read; use the bounce buffer */
            pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
-            if (!bounce_buffer) {
-                int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
-                int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
-                int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
-
-                bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
-                if (!bounce_buffer) {
-                    ret = -ENOMEM;
-                    goto err;
-                }
-            }
            qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);

            ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
-                                     &local_qiov, 0, 0);
+                                     &local_qiov, 0);
            if (ret < 0) {
                goto err;
            }
@@ -1379,7 +1283,7 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
                 * necessary to flush even in cache=writethrough mode.
                 */
                ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
-                                          &local_qiov, 0,
+                                          &local_qiov,
                                          BDRV_REQ_WRITE_UNCHANGED);
            }

@@ -1392,16 +1296,15 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
                goto err;
            }

-            if (!(flags & BDRV_REQ_PREFETCH)) {
-                qemu_iovec_from_buf(qiov, qiov_offset + progress,
-                                    bounce_buffer + skip_bytes,
-                                    pnum - skip_bytes);
-            }
-        } else if (!(flags & BDRV_REQ_PREFETCH)) {
+            qemu_iovec_from_buf(qiov, progress, bounce_buffer + skip_bytes,
+                                pnum - skip_bytes);
+        } else {
            /* Read directly into the destination */
-            ret = bdrv_driver_preadv(bs, offset + progress,
-                                     MIN(pnum - skip_bytes, bytes - progress),
-                                     qiov, qiov_offset + progress, 0);
+            qemu_iovec_init(&local_qiov, qiov->niov);
+            qemu_iovec_concat(&local_qiov, qiov, progress, pnum - skip_bytes);
+            ret = bdrv_driver_preadv(bs, offset + progress, local_qiov.size,
+                                     &local_qiov, 0);
+            qemu_iovec_destroy(&local_qiov);
            if (ret < 0) {
                goto err;
            }
@@ -1426,7 +1329,7 @@ err:
 */
 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
    BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
-    int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
+    int64_t align, QEMUIOVector *qiov, int flags)
 {
    BlockDriverState *bs = child->bs;
    int64_t total_bytes, max_bytes;
@@ -1437,6 +1340,7 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
    assert(is_power_of_2(align));
    assert((offset & (align - 1)) == 0);
    assert((bytes & (align - 1)) == 0);
+    assert(!qiov || bytes == qiov->size);
    assert((bs->open_flags & BDRV_O_NO_IO) == 0);
    max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
                                   align);
@@ -1445,8 +1349,7 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
     * potential fallback support, if we ever implement any read flags
     * to pass through to drivers.  For now, there aren't any
     * passthrough flags.  */
-    assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ |
-                       BDRV_REQ_PREFETCH)));
+    assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ)));

    /* Handle Copy on Read and associated serialisation */
    if (flags & BDRV_REQ_COPY_ON_READ) {
@@ -1474,10 +1377,7 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
        }

        if (!ret || pnum != bytes) {
-            ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
-                                           qiov, qiov_offset, flags);
-            goto out;
-        } else if (flags & BDRV_REQ_PREFETCH) {
+            ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov);
            goto out;
        }
    }
@@ -1491,7 +1391,7 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,

    max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
    if (bytes <= max_bytes && bytes <= max_transfer) {
-        ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0);
+        ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
        goto out;
    }

@@ -1499,12 +1399,17 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
        int num;

        if (max_bytes) {
+            QEMUIOVector local_qiov;
+
            num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
            assert(num);
+            qemu_iovec_init(&local_qiov, qiov->niov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);

            ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
-                                     num, qiov, bytes - bytes_remaining, 0);
+                                     num, &local_qiov, 0);
            max_bytes -= num;
+            qemu_iovec_destroy(&local_qiov);
        } else {
            num = bytes_remaining;
            ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0,
@@ -1606,7 +1511,7 @@ static int bdrv_padding_rmw_read(BdrvChild *child,
            bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
        }
        ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
-                                  align, &local_qiov, 0, 0);
+                                  align, &local_qiov, 0);
        if (ret < 0) {
            return ret;
        }
@@ -1629,7 +1534,7 @@ static int bdrv_padding_rmw_read(BdrvChild *child,
        ret = bdrv_aligned_preadv(
                child, req,
                req->overlap_offset + req->overlap_bytes - align,
-                align, align, &local_qiov, 0, 0);
+                align, align, &local_qiov, 0);
        if (ret < 0) {
            return ret;
        }
@@ -1664,8 +1569,7 @@ static void bdrv_padding_destroy(BdrvRequestPadding *pad)
 *
 * Function always succeeds.
 */
-static bool bdrv_pad_request(BlockDriverState *bs,
-                             QEMUIOVector **qiov, size_t *qiov_offset,
+static bool bdrv_pad_request(BlockDriverState *bs, QEMUIOVector **qiov,
                             int64_t *offset, unsigned int *bytes,
                             BdrvRequestPadding *pad)
 {
@@ -1674,12 +1578,11 @@ static bool bdrv_pad_request(BlockDriverState *bs,
    }

    qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
-                             *qiov, *qiov_offset, *bytes,
+                             *qiov, 0, *bytes,
                             pad->buf + pad->buf_len - pad->tail, pad->tail);
    *bytes += pad->head + pad->tail;
    *offset -= pad->head;
    *qiov = &pad->local_qiov;
-    *qiov_offset = 0;

    return true;
 }
@@ -1687,14 +1590,6 @@ static bool bdrv_pad_request(BlockDriverState *bs,
 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
    BdrvRequestFlags flags)
-{
-    return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
-}
-
-int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
-    int64_t offset, unsigned int bytes,
-    QEMUIOVector *qiov, size_t qiov_offset,
-    BdrvRequestFlags flags)
 {
    BlockDriverState *bs = child->bs;
    BdrvTrackedRequest req;
@@ -1715,12 +1610,12 @@ int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
        flags |= BDRV_REQ_COPY_ON_READ;
    }

-    bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad);
+    bdrv_pad_request(bs, &qiov, &offset, &bytes, &pad);

    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
    ret = bdrv_aligned_preadv(child, &req, offset, bytes,
                              bs->bl.request_alignment,
-                              qiov, qiov_offset, flags);
+                              qiov, flags);
    tracked_request_end(&req);
    bdrv_dec_in_flight(bs);

@@ -1796,7 +1691,7 @@ static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
            assert(!bs->supported_zero_flags);
        }

-        if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
+        if (ret < 0 && !(flags & BDRV_REQ_NO_FALLBACK)) {
            /* Fall back to bounce buffer if write zeroes is unsupported */
            BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;

@@ -1817,7 +1712,7 @@ static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
            }
            qemu_iovec_init_buf(&qiov, buf, num);

-            ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
+            ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags);

            /* Keep bounce buffer around if it is big enough for all
             * all future requests.
@@ -1932,7 +1827,7 @@ bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes,
 */
 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
    BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
-    int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
+    int64_t align, QEMUIOVector *qiov, int flags)
 {
    BlockDriverState *bs = child->bs;
    BlockDriver *drv = bs->drv;
@@ -1952,7 +1847,7 @@ static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
    assert(is_power_of_2(align));
    assert((offset & (align - 1)) == 0);
    assert((bytes & (align - 1)) == 0);
-    assert(!qiov || qiov_offset + bytes <= qiov->size);
+    assert(!qiov || bytes == qiov->size);
    max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
                                   align);

@@ -1960,7 +1855,7 @@ static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,

    if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
        !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
-        qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
+        qemu_iovec_is_zero(qiov, 0, qiov->size)) {
        flags |= BDRV_REQ_ZERO_WRITE;
        if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
            flags |= BDRV_REQ_MAY_UNMAP;
@@ -1973,15 +1868,15 @@ static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
        bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
        ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
    } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
-        ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
-                                             qiov, qiov_offset);
+        ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov);
    } else if (bytes <= max_transfer) {
        bdrv_debug_event(bs, BLKDBG_PWRITEV);
-        ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
+        ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
    } else {
        bdrv_debug_event(bs, BLKDBG_PWRITEV);
        while (bytes_remaining) {
            int num = MIN(bytes_remaining, max_transfer);
+            QEMUIOVector local_qiov;
            int local_flags = flags;

            assert(num);
@@ -1991,10 +1886,12 @@ static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
                 * need to flush on the last iteration */
                local_flags &= ~BDRV_REQ_FUA;
            }
+            qemu_iovec_init(&local_qiov, qiov->niov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);

            ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
-                                      num, qiov, bytes - bytes_remaining,
-                                      local_flags);
+                                      num, &local_qiov, local_flags);
+            qemu_iovec_destroy(&local_qiov);
            if (ret < 0) {
                break;
            }
@@ -2037,7 +1934,7 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,

            qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
            ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
-                                       align, &local_qiov, 0,
+                                       align, &local_qiov,
                                       flags & ~BDRV_REQ_ZERO_WRITE);
            if (ret < 0 || pad.merge_reads) {
                /* Error or all work is done */
@@ -2053,7 +1950,7 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
        /* Write the aligned part in the middle. */
        uint64_t aligned_bytes = bytes & ~(align - 1);
        ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
-                                   NULL, 0, flags);
+                                   NULL, flags);
        if (ret < 0) {
            goto out;
        }
@@ -2067,8 +1964,7 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,

        qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
        ret = bdrv_aligned_pwritev(child, req, offset, align, align,
-                                   &local_qiov, 0,
-                                   flags & ~BDRV_REQ_ZERO_WRITE);
+                                   &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
    }

 out:
@@ -2083,13 +1979,6 @@ out:
 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
    BdrvRequestFlags flags)
-{
-    return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
-}
-
-int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
-    int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset,
-    BdrvRequestFlags flags)
 {
    BlockDriverState *bs = child->bs;
    BdrvTrackedRequest req;
@@ -2108,13 +1997,6 @@ int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
        return ret;
    }

-    /* If the request is misaligned then we can't make it efficient */
-    if ((flags & BDRV_REQ_NO_FALLBACK) &&
-        !QEMU_IS_ALIGNED(offset | bytes, align))
-    {
-        return -ENOTSUP;
-    }
-
    bdrv_inc_in_flight(bs);
    /*
     * Align write if necessary by performing a read-modify-write cycle.
@@ -2128,14 +2010,14 @@ int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
        goto out;
    }

-    if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) {
+    if (bdrv_pad_request(bs, &qiov, &offset, &bytes, &pad)) {
        bdrv_mark_request_serialising(&req, align);
        bdrv_wait_serialising_requests(&req);
        bdrv_padding_rmw_read(child, &req, &pad, false);
    }

    ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
-                               qiov, qiov_offset, flags);
+                               qiov, flags);

    bdrv_padding_destroy(&pad);

@@ -2168,15 +2050,6 @@ int bdrv_flush_all(void)
    BlockDriverState *bs = NULL;
    int result = 0;

-    /*
-     * bdrv queue is managed by record/replay,
-     * creating new flush request for stopping
-     * the VM may break the determinism
-     */
-    if (replay_events_enabled()) {
-        return result;
-    }
-
    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
        AioContext *aio_context = bdrv_get_aio_context(bs);
        int ret;
@@ -3309,12 +3182,8 @@ static void bdrv_parent_cb_resize(BlockDriverState *bs)

 /**
 * Truncate file to 'offset' bytes (needed only for file protocols)
- *
- * If 'exact' is true, the file must be resized to exactly the given
- * 'offset'.  Otherwise, it is sufficient for the node to be at least
- * 'offset' bytes in length.
 */
-int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
+int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset,
                                  PreallocMode prealloc, Error **errp)
 {
    BlockDriverState *bs = child->bs;
@@ -3369,19 +3238,20 @@ int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
        goto out;
    }

-    if (drv->bdrv_co_truncate) {
-        ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, errp);
-    } else if (bs->file && drv->is_filter) {
-        ret = bdrv_co_truncate(bs->file, offset, exact, prealloc, errp);
-    } else {
+    if (!drv->bdrv_co_truncate) {
+        if (bs->file && drv->is_filter) {
+            ret = bdrv_co_truncate(bs->file, offset, prealloc, errp);
+            goto out;
+        }
        error_setg(errp, "Image format driver does not support resize");
        ret = -ENOTSUP;
        goto out;
    }
+
+    ret = drv->bdrv_co_truncate(bs, offset, prealloc, errp);
    if (ret < 0) {
        goto out;
    }
-
    ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
    if (ret < 0) {
        error_setg_errno(errp, -ret, "Could not refresh total sector count");
@@ -3403,7 +3273,6 @@ out:
 typedef struct TruncateCo {
    BdrvChild *child;
    int64_t offset;
-    bool exact;
    PreallocMode prealloc;
    Error **errp;
    int ret;
@@ -3412,19 +3281,18 @@ typedef struct TruncateCo {
 static void coroutine_fn bdrv_truncate_co_entry(void *opaque)
 {
    TruncateCo *tco = opaque;
-    tco->ret = bdrv_co_truncate(tco->child, tco->offset, tco->exact,
-                                tco->prealloc, tco->errp);
+    tco->ret = bdrv_co_truncate(tco->child, tco->offset, tco->prealloc,
+                                tco->errp);
    aio_wait_kick();
 }

-int bdrv_truncate(BdrvChild *child, int64_t offset, bool exact,
-                  PreallocMode prealloc, Error **errp)
+int bdrv_truncate(BdrvChild *child, int64_t offset, PreallocMode prealloc,
+                  Error **errp)
 {
    Coroutine *co;
    TruncateCo tco = {
        .child      = child,
        .offset     = offset,
-        .exact      = exact,
        .prealloc   = prealloc,
        .errp       = errp,
        .ret        = NOT_DONE,
@@ -40,7 +40,6 @@
 #include "qemu/module.h"
 #include "qemu/option.h"
 #include "qemu/uuid.h"
-#include "sysemu/replay.h"
 #include "qapi/error.h"
 #include "qapi/qapi-commands-misc.h"
 #include "qapi/qmp/qdict.h"
@@ -281,8 +280,8 @@ iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
    }

    if (iTask->co) {
-        replay_bh_schedule_oneshot_event(iTask->iscsilun->aio_context,
-                                         iscsi_co_generic_bh_cb, iTask);
+        aio_bh_schedule_oneshot(iTask->iscsilun->aio_context,
+                                 iscsi_co_generic_bh_cb, iTask);
    } else {
        iTask->complete = 1;
    }
@@ -2123,11 +2122,9 @@ static void iscsi_reopen_commit(BDRVReopenState *reopen_state)
 }

 static int coroutine_fn iscsi_co_truncate(BlockDriverState *bs, int64_t offset,
-                                          bool exact, PreallocMode prealloc,
-                                          Error **errp)
+                                          PreallocMode prealloc, Error **errp)
 {
    IscsiLun *iscsilun = bs->opaque;
-    int64_t cur_length;
    Error *local_err = NULL;

    if (prealloc != PREALLOC_MODE_OFF) {
@@ -2147,11 +2144,7 @@ static int coroutine_fn iscsi_co_truncate(BlockDriverState *bs, int64_t offset,
        return -EIO;
    }

-    cur_length = iscsi_getlength(bs);
-    if (offset != cur_length && exact) {
-        error_setg(errp, "Cannot resize iSCSI devices");
-        return -ENOTSUP;
-    } else if (offset > cur_length) {
+    if (offset > iscsi_getlength(bs)) {
        error_setg(errp, "Cannot grow iSCSI devices");
        return -EINVAL;
    }
@@ -51,8 +51,6 @@ typedef struct MirrorBlockJob {
    Error *replace_blocker;
    bool is_none_mode;
    BlockMirrorBackingMode backing_mode;
-    /* Whether the target image requires explicit zero-initialization */
-    bool zero_target;
    MirrorCopyMode copy_mode;
    BlockdevOnError on_source_error, on_target_error;
    bool synced;
@@ -478,7 +476,7 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
        int64_t next_offset = offset + nb_chunks * s->granularity;
        int64_t next_chunk = next_offset / s->granularity;
        if (next_offset >= s->bdev_length ||
-            !bdrv_dirty_bitmap_get_locked(s->dirty_bitmap, next_offset)) {
+            !bdrv_get_dirty_locked(source, s->dirty_bitmap, next_offset)) {
            break;
        }
        if (test_bit(next_chunk, s->in_flight_bitmap)) {
@@ -643,7 +641,7 @@ static int mirror_exit_common(Job *job)
        bdrv_unfreeze_backing_chain(mirror_top_bs, target_bs);
    }

-    bdrv_release_dirty_bitmap(s->dirty_bitmap);
+    bdrv_release_dirty_bitmap(src, s->dirty_bitmap);

    /* Make sure that the source BDS doesn't go away during bdrv_replace_node,
     * before we can call bdrv_drained_end */
@@ -651,11 +649,14 @@ static int mirror_exit_common(Job *job)
    bdrv_ref(mirror_top_bs);
    bdrv_ref(target_bs);

-    /*
-     * Remove target parent that still uses BLK_PERM_WRITE/RESIZE before
+    /* Remove target parent that still uses BLK_PERM_WRITE/RESIZE before
     * inserting target_bs at s->to_replace, where we might not be able to get
     * these permissions.
-     */
+     *
+     * Note that blk_unref() alone doesn't necessarily drop permissions because
+     * we might be running nested inside mirror_drain(), which takes an extra
+     * reference, so use an explicit blk_set_perm() first. */
+    blk_set_perm(s->target, 0, BLK_PERM_ALL, &error_abort);
    blk_unref(s->target);
    s->target = NULL;

@@ -771,7 +772,7 @@ static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
    int ret;
    int64_t count;

-    if (s->zero_target) {
+    if (base == NULL && !bdrv_has_zero_init(target_bs)) {
        if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
            bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
            return 0;
@@ -878,8 +879,8 @@ static int coroutine_fn mirror_run(Job *job, Error **errp)
        }

        if (s->bdev_length > base_length) {
-            ret = blk_truncate(s->target, s->bdev_length, false,
-                               PREALLOC_MODE_OFF, NULL);
+            ret = blk_truncate(s->target, s->bdev_length, PREALLOC_MODE_OFF,
+                               NULL);
            if (ret < 0) {
                goto immediate_exit;
            }
@@ -1151,12 +1152,28 @@ static bool mirror_drained_poll(BlockJob *job)
    return !!s->in_flight;
 }

+static void mirror_drain(BlockJob *job)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+
+    /* Need to keep a reference in case blk_drain triggers execution
+     * of mirror_complete...
+     */
+    if (s->target) {
+        BlockBackend *target = s->target;
+        blk_ref(target);
+        blk_drain(target);
+        blk_unref(target);
+    }
+}
+
 static const BlockJobDriver mirror_job_driver = {
    .job_driver = {
        .instance_size          = sizeof(MirrorBlockJob),
        .job_type               = JOB_TYPE_MIRROR,
        .free                   = block_job_free,
        .user_resume            = block_job_user_resume,
+        .drain                  = block_job_drain,
        .run                    = mirror_run,
        .prepare                = mirror_prepare,
        .abort                  = mirror_abort,
@@ -1164,6 +1181,7 @@ static const BlockJobDriver mirror_job_driver = {
        .complete               = mirror_complete,
    },
    .drained_poll           = mirror_drained_poll,
+    .drain                  = mirror_drain,
 };

 static const BlockJobDriver commit_active_job_driver = {
@@ -1172,6 +1190,7 @@ static const BlockJobDriver commit_active_job_driver = {
        .job_type               = JOB_TYPE_COMMIT,
        .free                   = block_job_free,
        .user_resume            = block_job_user_resume,
+        .drain                  = block_job_drain,
        .run                    = mirror_run,
        .prepare                = mirror_prepare,
        .abort                  = mirror_abort,
@@ -1179,6 +1198,7 @@ static const BlockJobDriver commit_active_job_driver = {
        .complete               = mirror_complete,
    },
    .drained_poll           = mirror_drained_poll,
+    .drain                  = mirror_drain,
 };

 static void coroutine_fn
@@ -1186,107 +1206,84 @@ do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
                     uint64_t offset, uint64_t bytes,
                     QEMUIOVector *qiov, int flags)
 {
-    int ret;
-    size_t qiov_offset = 0;
-    int64_t bitmap_offset, bitmap_end;
+    QEMUIOVector target_qiov;
+    uint64_t dirty_offset = offset;
+    uint64_t dirty_bytes;

-    if (!QEMU_IS_ALIGNED(offset, job->granularity) &&
-        bdrv_dirty_bitmap_get(job->dirty_bitmap, offset))
-    {
-            /*
-             * Dirty unaligned padding: ignore it.
-             *
-             * Reasoning:
-             * 1. If we copy it, we can't reset corresponding bit in
-             *    dirty_bitmap as there may be some "dirty" bytes still not
-             *    copied.
-             * 2. It's already dirty, so skipping it we don't diverge mirror
-             *    progress.
-             *
-             * Note, that because of this, guest write may have no contribution
-             * into mirror converge, but that's not bad, as we have background
-             * process of mirroring. If under some bad circumstances (high guest
-             * IO load) background process starve, we will not converge anyway,
-             * even if each write will contribute, as guest is not guaranteed to
-             * rewrite the whole disk.
-             */
-            qiov_offset = QEMU_ALIGN_UP(offset, job->granularity) - offset;
-            if (bytes <= qiov_offset) {
-                /* nothing to do after shrink */
-                return;
-            }
-            offset += qiov_offset;
-            bytes -= qiov_offset;
+    if (qiov) {
+        qemu_iovec_init(&target_qiov, qiov->niov);
    }

-    if (!QEMU_IS_ALIGNED(offset + bytes, job->granularity) &&
-        bdrv_dirty_bitmap_get(job->dirty_bitmap, offset + bytes - 1))
-    {
-        uint64_t tail = (offset + bytes) % job->granularity;
+    while (true) {
+        bool valid_area;
+        int ret;

-        if (bytes <= tail) {
-            /* nothing to do after shrink */
-            return;
+        bdrv_dirty_bitmap_lock(job->dirty_bitmap);
+        dirty_bytes = MIN(offset + bytes - dirty_offset, INT_MAX);
+        valid_area = bdrv_dirty_bitmap_next_dirty_area(job->dirty_bitmap,
+                                                       &dirty_offset,
+                                                       &dirty_bytes);
+        if (!valid_area) {
+            bdrv_dirty_bitmap_unlock(job->dirty_bitmap);
+            break;
        }
-        bytes -= tail;
-    }

-    /*
-     * Tails are either clean or shrunk, so for bitmap resetting
-     * we safely align the range down.
-     */
-    bitmap_offset = QEMU_ALIGN_UP(offset, job->granularity);
-    bitmap_end = QEMU_ALIGN_DOWN(offset + bytes, job->granularity);
-    if (bitmap_offset < bitmap_end) {
-        bdrv_reset_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
-                                bitmap_end - bitmap_offset);
-    }
+        bdrv_reset_dirty_bitmap_locked(job->dirty_bitmap,
+                                       dirty_offset, dirty_bytes);
+        bdrv_dirty_bitmap_unlock(job->dirty_bitmap);

-    job_progress_increase_remaining(&job->common.job, bytes);
+        job_progress_increase_remaining(&job->common.job, dirty_bytes);

-    switch (method) {
-    case MIRROR_METHOD_COPY:
-        ret = blk_co_pwritev_part(job->target, offset, bytes,
-                                  qiov, qiov_offset, flags);
-        break;
+        assert(dirty_offset - offset <= SIZE_MAX);
+        if (qiov) {
+            qemu_iovec_reset(&target_qiov);
+            qemu_iovec_concat(&target_qiov, qiov,
+                              dirty_offset - offset, dirty_bytes);
+        }

-    case MIRROR_METHOD_ZERO:
-        assert(!qiov);
-        ret = blk_co_pwrite_zeroes(job->target, offset, bytes, flags);
-        break;
+        switch (method) {
+        case MIRROR_METHOD_COPY:
+            ret = blk_co_pwritev(job->target, dirty_offset, dirty_bytes,
+                                 qiov ? &target_qiov : NULL, flags);
+            break;

-    case MIRROR_METHOD_DISCARD:
-        assert(!qiov);
-        ret = blk_co_pdiscard(job->target, offset, bytes);
-        break;
+        case MIRROR_METHOD_ZERO:
+            assert(!qiov);
+            ret = blk_co_pwrite_zeroes(job->target, dirty_offset, dirty_bytes,
+                                       flags);
+            break;

-    default:
-        abort();
-    }
+        case MIRROR_METHOD_DISCARD:
+            assert(!qiov);
+            ret = blk_co_pdiscard(job->target, dirty_offset, dirty_bytes);
+            break;

-    if (ret >= 0) {
-        job_progress_update(&job->common.job, bytes);
-    } else {
-        BlockErrorAction action;
+        default:
+            abort();
+        }

-        /*
-         * We failed, so we should mark dirty the whole area, aligned up.
-         * Note that we don't care about shrunk tails if any: they were dirty
-         * at function start, and they must be still dirty, as we've locked
-         * the region for in-flight op.
-         */
-        bitmap_offset = QEMU_ALIGN_DOWN(offset, job->granularity);
-        bitmap_end = QEMU_ALIGN_UP(offset + bytes, job->granularity);
-        bdrv_set_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
-                              bitmap_end - bitmap_offset);
-        job->actively_synced = false;
+        if (ret >= 0) {
+            job_progress_update(&job->common.job, dirty_bytes);
+        } else {
+            BlockErrorAction action;

-        action = mirror_error_action(job, false, -ret);
-        if (action == BLOCK_ERROR_ACTION_REPORT) {
-            if (!job->ret) {
-                job->ret = ret;
+            bdrv_set_dirty_bitmap(job->dirty_bitmap, dirty_offset, dirty_bytes);
+            job->actively_synced = false;
+
+            action = mirror_error_action(job, false, -ret);
+            if (action == BLOCK_ERROR_ACTION_REPORT) {
+                if (!job->ret) {
+                    job->ret = ret;
+                }
+                break;
            }
        }
+
+        dirty_offset += dirty_bytes;
+    }
+
+    if (qiov) {
+        qemu_iovec_destroy(&target_qiov);
    }
 }

@@ -1493,6 +1490,15 @@ static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
    *nshared = BLK_PERM_ALL;
 }

+static void bdrv_mirror_top_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    MirrorBDSOpaque *s = bs->opaque;
+
+    if (s && s->job && s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
+        bs->bl.request_alignment = s->job->granularity;
+    }
+}
+
 /* Dummy node that provides consistent read to its users without requiring it
 * from its backing file and that allows writes on the backing file chain. */
 static BlockDriver bdrv_mirror_top = {
@@ -1505,6 +1511,7 @@ static BlockDriver bdrv_mirror_top = {
    .bdrv_co_block_status       = bdrv_co_block_status_from_backing,
    .bdrv_refresh_filename      = bdrv_mirror_top_refresh_filename,
    .bdrv_child_perm            = bdrv_mirror_top_child_perm,
+    .bdrv_refresh_limits        = bdrv_mirror_top_refresh_limits,
 };

 static BlockJob *mirror_start_job(
@@ -1513,7 +1520,6 @@ static BlockJob *mirror_start_job(
                             const char *replaces, int64_t speed,
                             uint32_t granularity, int64_t buf_size,
                             BlockMirrorBackingMode backing_mode,
-                             bool zero_target,
                             BlockdevOnError on_source_error,
                             BlockdevOnError on_target_error,
                             bool unmap,
@@ -1635,14 +1641,12 @@ static BlockJob *mirror_start_job(
        blk_set_force_allow_inactivate(s->target);
    }
    blk_set_allow_aio_context_change(s->target, true);
-    blk_set_disable_request_queuing(s->target, true);

    s->replaces = g_strdup(replaces);
    s->on_source_error = on_source_error;
    s->on_target_error = on_target_error;
    s->is_none_mode = is_none_mode;
    s->backing_mode = backing_mode;
-    s->zero_target = zero_target;
    s->copy_mode = copy_mode;
    s->base = base;
    s->granularity = granularity;
@@ -1652,13 +1656,29 @@ static BlockJob *mirror_start_job(
        s->should_complete = true;
    }

+    /*
+     * Must be called before we start tracking writes, but after
+     *
+     *     ((MirrorBlockJob *)
+     *         ((MirrorBDSOpaque *)
+     *             mirror_top_bs->opaque
+     *         )->job
+     *     )->copy_mode
+     *
+     * has the correct value.
+     * (We start tracking writes as of the following
+     * bdrv_create_dirty_bitmap() call.)
+     */
+    bdrv_refresh_limits(mirror_top_bs, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
    s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
    if (!s->dirty_bitmap) {
        goto fail;
    }
-    if (s->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
-        bdrv_disable_dirty_bitmap(s->dirty_bitmap);
-    }

    ret = block_job_add_bdrv(&s->common, "source", bs, 0,
                             BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
@@ -1711,7 +1731,7 @@ fail:
        blk_unref(s->target);
        bs_opaque->job = NULL;
        if (s->dirty_bitmap) {
-            bdrv_release_dirty_bitmap(s->dirty_bitmap);
+            bdrv_release_dirty_bitmap(bs, s->dirty_bitmap);
        }
        job_early_fail(&s->common.job);
    }
@@ -1731,7 +1751,6 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
                  int creation_flags, int64_t speed,
                  uint32_t granularity, int64_t buf_size,
                  MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
-                  bool zero_target,
                  BlockdevOnError on_source_error,
                  BlockdevOnError on_target_error,
                  bool unmap, const char *filter_node_name,
@@ -1740,16 +1759,14 @@ void mirror_start(const char *job_id, BlockDriverState *bs,
    bool is_none_mode;
    BlockDriverState *base;

-    if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
-        (mode == MIRROR_SYNC_MODE_BITMAP)) {
-        error_setg(errp, "Sync mode '%s' not supported",
-                   MirrorSyncMode_str(mode));
+    if (mode == MIRROR_SYNC_MODE_INCREMENTAL) {
+        error_setg(errp, "Sync mode 'incremental' not supported");
        return;
    }
    is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
    base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
    mirror_start_job(job_id, bs, creation_flags, target, replaces,
-                     speed, granularity, buf_size, backing_mode, zero_target,
+                     speed, granularity, buf_size, backing_mode,
                     on_source_error, on_target_error, unmap, NULL, NULL,
                     &mirror_job_driver, is_none_mode, base, false,
                     filter_node_name, true, copy_mode, errp);
@@ -1776,7 +1793,7 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,

    ret = mirror_start_job(
                     job_id, bs, creation_flags, base, NULL, speed, 0, 0,
-                     MIRROR_LEAVE_BACKING_CHAIN, false,
+                     MIRROR_LEAVE_BACKING_CHAIN,
                     on_error, on_error, true, cb, opaque,
                     &commit_active_job_driver, false, base, auto_complete,
                     filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
@@ -1,7 +1,6 @@
 /*
 * QEMU Block driver for  NBD
 *
- * Copyright (c) 2019 Virtuozzo International GmbH.
 * Copyright (C) 2016 Red Hat, Inc.
 * Copyright (C) 2008 Bull S.A.S.
 *     Author: Laurent Vivier <Laurent.Vivier@bull.net>
@@ -34,7 +33,6 @@
 #include "qemu/uri.h"
 #include "qemu/option.h"
 #include "qemu/cutils.h"
-#include "qemu/main-loop.h"

 #include "qapi/qapi-visit-sockets.h"
 #include "qapi/qmp/qstring.h"
@@ -55,13 +53,6 @@ typedef struct {
    bool receiving;         /* waiting for connection_co? */
 } NBDClientRequest;

-typedef enum NBDClientState {
-    NBD_CLIENT_CONNECTING_WAIT,
-    NBD_CLIENT_CONNECTING_NOWAIT,
-    NBD_CLIENT_CONNECTED,
-    NBD_CLIENT_QUIT
-} NBDClientState;
-
 typedef struct BDRVNBDState {
    QIOChannelSocket *sioc; /* The master data channel */
    QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
@@ -70,45 +61,18 @@ typedef struct BDRVNBDState {
    CoMutex send_mutex;
    CoQueue free_sema;
    Coroutine *connection_co;
-    QemuCoSleepState *connection_co_sleep_ns_state;
-    bool drained;
-    bool wait_drained_end;
    int in_flight;
-    NBDClientState state;
-    int connect_status;
-    Error *connect_err;
-    bool wait_in_flight;

    NBDClientRequest requests[MAX_NBD_REQUESTS];
    NBDReply reply;
    BlockDriverState *bs;
+    bool quit;

-    /* Connection parameters */
-    uint32_t reconnect_delay;
+    /* For nbd_refresh_filename() */
    SocketAddress *saddr;
    char *export, *tlscredsid;
-    QCryptoTLSCreds *tlscreds;
-    const char *hostname;
-    char *x_dirty_bitmap;
 } BDRVNBDState;

-static int nbd_client_connect(BlockDriverState *bs, Error **errp);
-
-static void nbd_channel_error(BDRVNBDState *s, int ret)
-{
-    if (ret == -EIO) {
-        if (s->state == NBD_CLIENT_CONNECTED) {
-            s->state = s->reconnect_delay ? NBD_CLIENT_CONNECTING_WAIT :
-                                            NBD_CLIENT_CONNECTING_NOWAIT;
-        }
-    } else {
-        if (s->state == NBD_CLIENT_CONNECTED) {
-            qio_channel_shutdown(s->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
-        }
-        s->state = NBD_CLIENT_QUIT;
-    }
-}
-
 static void nbd_recv_coroutines_wake_all(BDRVNBDState *s)
 {
    int i;
@@ -149,13 +113,7 @@ static void nbd_client_attach_aio_context(BlockDriverState *bs,
 {
    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;

-    /*
-     * s->connection_co is either yielded from nbd_receive_reply or from
-     * nbd_co_reconnect_loop()
-     */
-    if (s->state == NBD_CLIENT_CONNECTED) {
-        qio_channel_attach_aio_context(QIO_CHANNEL(s->ioc), new_context);
-    }
+    qio_channel_attach_aio_context(QIO_CHANNEL(s->ioc), new_context);

    bdrv_inc_in_flight(bs);

@@ -166,150 +124,24 @@ static void nbd_client_attach_aio_context(BlockDriverState *bs,
    aio_wait_bh_oneshot(new_context, nbd_client_attach_aio_context_bh, bs);
 }

-static void coroutine_fn nbd_client_co_drain_begin(BlockDriverState *bs)
-{
-    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
-
-    s->drained = true;
-    if (s->connection_co_sleep_ns_state) {
-        qemu_co_sleep_wake(s->connection_co_sleep_ns_state);
-    }
-}
-
-static void coroutine_fn nbd_client_co_drain_end(BlockDriverState *bs)
-{
-    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
-
-    s->drained = false;
-    if (s->wait_drained_end) {
-        s->wait_drained_end = false;
-        aio_co_wake(s->connection_co);
-    }
-}
-

 static void nbd_teardown_connection(BlockDriverState *bs)
 {
    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;

-    if (s->state == NBD_CLIENT_CONNECTED) {
-        /* finish any pending coroutines */
-        assert(s->ioc);
-        qio_channel_shutdown(s->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL);
-    }
-    s->state = NBD_CLIENT_QUIT;
-    if (s->connection_co) {
-        if (s->connection_co_sleep_ns_state) {
-            qemu_co_sleep_wake(s->connection_co_sleep_ns_state);
-        }
-    }
+    assert(s->ioc);
+
+    /* finish any pending coroutines */
+    qio_channel_shutdown(s->ioc,
+                         QIO_CHANNEL_SHUTDOWN_BOTH,
+                         NULL);
    BDRV_POLL_WHILE(bs, s->connection_co);
-}

-static bool nbd_client_connecting(BDRVNBDState *s)
-{
-    return s->state == NBD_CLIENT_CONNECTING_WAIT ||
-        s->state == NBD_CLIENT_CONNECTING_NOWAIT;
-}
-
-static bool nbd_client_connecting_wait(BDRVNBDState *s)
-{
-    return s->state == NBD_CLIENT_CONNECTING_WAIT;
-}
-
-static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
-{
-    Error *local_err = NULL;
-
-    if (!nbd_client_connecting(s)) {
-        return;
-    }
-
-    /* Wait for completion of all in-flight requests */
-
-    qemu_co_mutex_lock(&s->send_mutex);
-
-    while (s->in_flight > 0) {
-        qemu_co_mutex_unlock(&s->send_mutex);
-        nbd_recv_coroutines_wake_all(s);
-        s->wait_in_flight = true;
-        qemu_coroutine_yield();
-        s->wait_in_flight = false;
-        qemu_co_mutex_lock(&s->send_mutex);
-    }
-
-    qemu_co_mutex_unlock(&s->send_mutex);
-
-    if (!nbd_client_connecting(s)) {
-        return;
-    }
-
-    /*
-     * Now we are sure that nobody is accessing the channel, and no one will
-     * try until we set the state to CONNECTED.
-     */
-
-    /* Finalize previous connection if any */
-    if (s->ioc) {
-        nbd_client_detach_aio_context(s->bs);
-        object_unref(OBJECT(s->sioc));
-        s->sioc = NULL;
-        object_unref(OBJECT(s->ioc));
-        s->ioc = NULL;
-    }
-
-    s->connect_status = nbd_client_connect(s->bs, &local_err);
-    error_free(s->connect_err);
-    s->connect_err = NULL;
-    error_propagate(&s->connect_err, local_err);
-
-    if (s->connect_status < 0) {
-        /* failed attempt */
-        return;
-    }
-
-    /* successfully connected */
-    s->state = NBD_CLIENT_CONNECTED;
-    qemu_co_queue_restart_all(&s->free_sema);
-}
-
-static coroutine_fn void nbd_co_reconnect_loop(BDRVNBDState *s)
-{
-    uint64_t start_time_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-    uint64_t delay_ns = s->reconnect_delay * NANOSECONDS_PER_SECOND;
-    uint64_t timeout = 1 * NANOSECONDS_PER_SECOND;
-    uint64_t max_timeout = 16 * NANOSECONDS_PER_SECOND;
-
-    nbd_reconnect_attempt(s);
-
-    while (nbd_client_connecting(s)) {
-        if (s->state == NBD_CLIENT_CONNECTING_WAIT &&
-            qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time_ns > delay_ns)
-        {
-            s->state = NBD_CLIENT_CONNECTING_NOWAIT;
-            qemu_co_queue_restart_all(&s->free_sema);
-        }
-
-        qemu_co_sleep_ns_wakeable(QEMU_CLOCK_REALTIME, timeout,
-                                  &s->connection_co_sleep_ns_state);
-        if (s->drained) {
-            bdrv_dec_in_flight(s->bs);
-            s->wait_drained_end = true;
-            while (s->drained) {
-                /*
-                 * We may be entered once from nbd_client_attach_aio_context_bh
-                 * and then from nbd_client_co_drain_end. So here is a loop.
-                 */
-                qemu_coroutine_yield();
-            }
-            bdrv_inc_in_flight(s->bs);
-        }
-        if (timeout < max_timeout) {
-            timeout *= 2;
-        }
-
-        nbd_reconnect_attempt(s);
-    }
+    nbd_client_detach_aio_context(bs);
+    object_unref(OBJECT(s->sioc));
+    s->sioc = NULL;
+    object_unref(OBJECT(s->ioc));
+    s->ioc = NULL;
 }

 static coroutine_fn void nbd_connection_entry(void *opaque)
@@ -319,7 +151,7 @@ static coroutine_fn void nbd_connection_entry(void *opaque)
    int ret = 0;
    Error *local_err = NULL;

-    while (s->state != NBD_CLIENT_QUIT) {
+    while (!s->quit) {
        /*
         * The NBD client can only really be considered idle when it has
         * yielded from qio_channel_readv_all_eof(), waiting for data. This is
@@ -329,26 +161,15 @@ static coroutine_fn void nbd_connection_entry(void *opaque)
         * Therefore we keep an additional in_flight reference all the time and
         * only drop it temporarily here.
         */
-
-        if (nbd_client_connecting(s)) {
-            nbd_co_reconnect_loop(s);
-        }
-
-        if (s->state != NBD_CLIENT_CONNECTED) {
-            continue;
-        }
-
        assert(s->reply.handle == 0);
        ret = nbd_receive_reply(s->bs, s->ioc, &s->reply, &local_err);

        if (local_err) {
            trace_nbd_read_reply_entry_fail(ret, error_get_pretty(local_err));
            error_free(local_err);
-            local_err = NULL;
        }
        if (ret <= 0) {
-            nbd_channel_error(s, ret ? ret : -EIO);
-            continue;
+            break;
        }

        /*
@@ -362,8 +183,7 @@ static coroutine_fn void nbd_connection_entry(void *opaque)
            !s->requests[i].receiving ||
            (nbd_reply_is_structured(&s->reply) && !s->info.structured_reply))
        {
-            nbd_channel_error(s, -EINVAL);
-            continue;
+            break;
        }

        /*
@@ -382,19 +202,11 @@ static coroutine_fn void nbd_connection_entry(void *opaque)
        qemu_coroutine_yield();
    }

-    qemu_co_queue_restart_all(&s->free_sema);
+    s->quit = true;
    nbd_recv_coroutines_wake_all(s);
    bdrv_dec_in_flight(s->bs);

    s->connection_co = NULL;
-    if (s->ioc) {
-        nbd_client_detach_aio_context(s->bs);
-        object_unref(OBJECT(s->sioc));
-        s->sioc = NULL;
-        object_unref(OBJECT(s->ioc));
-        s->ioc = NULL;
-    }
-
    aio_wait_kick();
 }

@@ -403,18 +215,12 @@ static int nbd_co_send_request(BlockDriverState *bs,
                               QEMUIOVector *qiov)
 {
    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
-    int rc, i = -1;
+    int rc, i;

    qemu_co_mutex_lock(&s->send_mutex);
-    while (s->in_flight == MAX_NBD_REQUESTS || nbd_client_connecting_wait(s)) {
+    while (s->in_flight == MAX_NBD_REQUESTS) {
        qemu_co_queue_wait(&s->free_sema, &s->send_mutex);
    }
-
-    if (s->state != NBD_CLIENT_CONNECTED) {
-        rc = -EIO;
-        goto err;
-    }
-
    s->in_flight++;

    for (i = 0; i < MAX_NBD_REQUESTS; i++) {
@@ -432,12 +238,16 @@ static int nbd_co_send_request(BlockDriverState *bs,

    request->handle = INDEX_TO_HANDLE(s, i);

+    if (s->quit) {
+        rc = -EIO;
+        goto err;
+    }
    assert(s->ioc);

    if (qiov) {
        qio_channel_set_cork(s->ioc, true);
        rc = nbd_send_request(s->ioc, request);
-        if (rc >= 0 && s->state == NBD_CLIENT_CONNECTED) {
+        if (rc >= 0 && !s->quit) {
            if (qio_channel_writev_all(s->ioc, qiov->iov, qiov->niov,
                                       NULL) < 0) {
                rc = -EIO;
@@ -452,16 +262,10 @@ static int nbd_co_send_request(BlockDriverState *bs,

 err:
    if (rc < 0) {
-        nbd_channel_error(s, rc);
-        if (i != -1) {
-            s->requests[i].coroutine = NULL;
-            s->in_flight--;
-        }
-        if (s->in_flight == 0 && s->wait_in_flight) {
-            aio_co_wake(s->connection_co);
-        } else {
-            qemu_co_queue_next(&s->free_sema);
-        }
+        s->quit = true;
+        s->requests[i].coroutine = NULL;
+        s->in_flight--;
+        qemu_co_queue_next(&s->free_sema);
    }
    qemu_co_mutex_unlock(&s->send_mutex);
    return rc;
@@ -752,7 +556,7 @@ static coroutine_fn int nbd_co_do_receive_one_chunk(
    s->requests[i].receiving = true;
    qemu_coroutine_yield();
    s->requests[i].receiving = false;
-    if (s->state != NBD_CLIENT_CONNECTED) {
+    if (s->quit) {
        error_setg(errp, "Connection closed");
        return -EIO;
    }
@@ -837,19 +641,14 @@ static coroutine_fn int nbd_co_receive_one_chunk(

    if (ret < 0) {
        memset(reply, 0, sizeof(*reply));
-        nbd_channel_error(s, ret);
+        s->quit = true;
    } else {
        /* For assert at loop start in nbd_connection_entry */
        *reply = s->reply;
+        s->reply.handle = 0;
    }
-    s->reply.handle = 0;

-    if (s->connection_co && !s->wait_in_flight) {
-        /*
-         * We must check s->wait_in_flight, because we may entered by
-         * nbd_recv_coroutines_wake_all(), in this case we should not
-         * wake connection_co here, it will woken by last request.
-         */
+    if (s->connection_co) {
        aio_co_wake(s->connection_co);
    }

@@ -910,7 +709,7 @@ static bool nbd_reply_chunk_iter_receive(BDRVNBDState *s,
    NBDReply local_reply;
    NBDStructuredReplyChunk *chunk;
    Error *local_err = NULL;
-    if (s->state != NBD_CLIENT_CONNECTED) {
+    if (s->quit) {
        error_setg(&local_err, "Connection closed");
        nbd_iter_channel_error(iter, -EIO, &local_err);
        goto break_loop;
@@ -935,7 +734,7 @@ static bool nbd_reply_chunk_iter_receive(BDRVNBDState *s,
    }

    /* Do not execute the body of NBD_FOREACH_REPLY_CHUNK for simple reply. */
-    if (nbd_reply_is_simple(reply) || s->state != NBD_CLIENT_CONNECTED) {
+    if (nbd_reply_is_simple(reply) || s->quit) {
        goto break_loop;
    }

@@ -961,11 +760,7 @@ break_loop:

    qemu_co_mutex_lock(&s->send_mutex);
    s->in_flight--;
-    if (s->in_flight == 0 && s->wait_in_flight) {
-        aio_co_wake(s->connection_co);
-    } else {
-        qemu_co_queue_next(&s->free_sema);
-    }
+    qemu_co_queue_next(&s->free_sema);
    qemu_co_mutex_unlock(&s->send_mutex);

    return false;
@@ -1013,14 +808,14 @@ static int nbd_co_receive_cmdread_reply(BDRVNBDState *s, uint64_t handle,
            ret = nbd_parse_offset_hole_payload(s, &reply.structured, payload,
                                                offset, qiov, &local_err);
            if (ret < 0) {
-                nbd_channel_error(s, ret);
+                s->quit = true;
                nbd_iter_channel_error(&iter, ret, &local_err);
            }
            break;
        default:
            if (!nbd_reply_type_is_error(chunk->type)) {
                /* not allowed reply type */
-                nbd_channel_error(s, -EINVAL);
+                s->quit = true;
                error_setg(&local_err,
                           "Unexpected reply type: %d (%s) for CMD_READ",
                           chunk->type, nbd_reply_type_lookup(chunk->type));
@@ -1058,7 +853,7 @@ static int nbd_co_receive_blockstatus_reply(BDRVNBDState *s,
        switch (chunk->type) {
        case NBD_REPLY_TYPE_BLOCK_STATUS:
            if (received) {
-                nbd_channel_error(s, -EINVAL);
+                s->quit = true;
                error_setg(&local_err, "Several BLOCK_STATUS chunks in reply");
                nbd_iter_channel_error(&iter, -EINVAL, &local_err);
            }
@@ -1068,13 +863,13 @@ static int nbd_co_receive_blockstatus_reply(BDRVNBDState *s,
                                                payload, length, extent,
                                                &local_err);
            if (ret < 0) {
-                nbd_channel_error(s, ret);
+                s->quit = true;
                nbd_iter_channel_error(&iter, ret, &local_err);
            }
            break;
        default:
            if (!nbd_reply_type_is_error(chunk->type)) {
-                nbd_channel_error(s, -EINVAL);
+                s->quit = true;
                error_setg(&local_err,
                           "Unexpected reply type: %d (%s) "
                           "for CMD_BLOCK_STATUS",
@@ -1111,26 +906,20 @@ static int nbd_co_request(BlockDriverState *bs, NBDRequest *request,
    } else {
        assert(request->type != NBD_CMD_WRITE);
    }
+    ret = nbd_co_send_request(bs, request, write_qiov);
+    if (ret < 0) {
+        return ret;
+    }

-    do {
-        ret = nbd_co_send_request(bs, request, write_qiov);
-        if (ret < 0) {
-            continue;
-        }
-
-        ret = nbd_co_receive_return_code(s, request->handle,
-                                         &request_ret, &local_err);
-        if (local_err) {
-            trace_nbd_co_request_fail(request->from, request->len,
-                                      request->handle, request->flags,
-                                      request->type,
-                                      nbd_cmd_lookup(request->type),
-                                      ret, error_get_pretty(local_err));
-            error_free(local_err);
-            local_err = NULL;
-        }
-    } while (ret < 0 && nbd_client_connecting_wait(s));
-
+    ret = nbd_co_receive_return_code(s, request->handle,
+                                     &request_ret, &local_err);
+    if (local_err) {
+        trace_nbd_co_request_fail(request->from, request->len, request->handle,
+                                  request->flags, request->type,
+                                  nbd_cmd_lookup(request->type),
+                                  ret, error_get_pretty(local_err));
+        error_free(local_err);
+    }
    return ret ? ret : request_ret;
 }

@@ -1171,24 +960,20 @@ static int nbd_client_co_preadv(BlockDriverState *bs, uint64_t offset,
        request.len -= slop;
    }

-    do {
-        ret = nbd_co_send_request(bs, &request, NULL);
-        if (ret < 0) {
-            continue;
-        }
-
-        ret = nbd_co_receive_cmdread_reply(s, request.handle, offset, qiov,
-                                           &request_ret, &local_err);
-        if (local_err) {
-            trace_nbd_co_request_fail(request.from, request.len, request.handle,
-                                      request.flags, request.type,
-                                      nbd_cmd_lookup(request.type),
-                                      ret, error_get_pretty(local_err));
-            error_free(local_err);
-            local_err = NULL;
-        }
-    } while (ret < 0 && nbd_client_connecting_wait(s));
+    ret = nbd_co_send_request(bs, &request, NULL);
+    if (ret < 0) {
+        return ret;
+    }

+    ret = nbd_co_receive_cmdread_reply(s, request.handle, offset, qiov,
+                                       &request_ret, &local_err);
+    if (local_err) {
+        trace_nbd_co_request_fail(request.from, request.len, request.handle,
+                                  request.flags, request.type,
+                                  nbd_cmd_lookup(request.type),
+                                  ret, error_get_pretty(local_err));
+        error_free(local_err);
+    }
    return ret ? ret : request_ret;
 }

@@ -1238,10 +1023,6 @@ static int nbd_client_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset,
    if (!(flags & BDRV_REQ_MAY_UNMAP)) {
        request.flags |= NBD_CMD_FLAG_NO_HOLE;
    }
-    if (flags & BDRV_REQ_NO_FALLBACK) {
-        assert(s->info.flags & NBD_FLAG_SEND_FAST_ZERO);
-        request.flags |= NBD_CMD_FLAG_FAST_ZERO;
-    }

    if (!bytes) {
        return 0;
@@ -1325,25 +1106,20 @@ static int coroutine_fn nbd_client_co_block_status(
    if (s->info.min_block) {
        assert(QEMU_IS_ALIGNED(request.len, s->info.min_block));
    }
-    do {
-        ret = nbd_co_send_request(bs, &request, NULL);
-        if (ret < 0) {
-            continue;
-        }
-
-        ret = nbd_co_receive_blockstatus_reply(s, request.handle, bytes,
-                                               &extent, &request_ret,
-                                               &local_err);
-        if (local_err) {
-            trace_nbd_co_request_fail(request.from, request.len, request.handle,
-                                      request.flags, request.type,
-                                      nbd_cmd_lookup(request.type),
-                                      ret, error_get_pretty(local_err));
-            error_free(local_err);
-            local_err = NULL;
-        }
-    } while (ret < 0 && nbd_client_connecting_wait(s));
+    ret = nbd_co_send_request(bs, &request, NULL);
+    if (ret < 0) {
+        return ret;
+    }

+    ret = nbd_co_receive_blockstatus_reply(s, request.handle, bytes,
+                                           &extent, &request_ret, &local_err);
+    if (local_err) {
+        trace_nbd_co_request_fail(request.from, request.len, request.handle,
+                                  request.flags, request.type,
+                                  nbd_cmd_lookup(request.type),
+                                  ret, error_get_pretty(local_err));
+        error_free(local_err);
+    }
    if (ret < 0 || request_ret < 0) {
        return ret ? ret : request_ret;
    }
@@ -1357,26 +1133,14 @@ static int coroutine_fn nbd_client_co_block_status(
        BDRV_BLOCK_OFFSET_VALID;
 }

-static int nbd_client_reopen_prepare(BDRVReopenState *state,
-                                     BlockReopenQueue *queue, Error **errp)
-{
-    BDRVNBDState *s = (BDRVNBDState *)state->bs->opaque;
-
-    if ((state->flags & BDRV_O_RDWR) && (s->info.flags & NBD_FLAG_READ_ONLY)) {
-        error_setg(errp, "Can't reopen read-only NBD mount as read/write");
-        return -EACCES;
-    }
-    return 0;
-}
-
 static void nbd_client_close(BlockDriverState *bs)
 {
    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
    NBDRequest request = { .type = NBD_CMD_DISC };

-    if (s->ioc) {
-        nbd_send_request(s->ioc, &request);
-    }
+    assert(s->ioc);
+
+    nbd_send_request(s->ioc, &request);

    nbd_teardown_connection(bs);
 }
@@ -1402,43 +1166,47 @@ static QIOChannelSocket *nbd_establish_connection(SocketAddress *saddr,
    return sioc;
 }

-static int nbd_client_connect(BlockDriverState *bs, Error **errp)
+static int nbd_client_connect(BlockDriverState *bs,
+                              SocketAddress *saddr,
+                              const char *export,
+                              QCryptoTLSCreds *tlscreds,
+                              const char *hostname,
+                              const char *x_dirty_bitmap,
+                              Error **errp)
 {
    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
-    AioContext *aio_context = bdrv_get_aio_context(bs);
    int ret;

    /*
     * establish TCP connection, return error if it fails
     * TODO: Configurable retry-until-timeout behaviour.
     */
-    QIOChannelSocket *sioc = nbd_establish_connection(s->saddr, errp);
+    QIOChannelSocket *sioc = nbd_establish_connection(saddr, errp);

    if (!sioc) {
        return -ECONNREFUSED;
    }

    /* NBD handshake */
-    trace_nbd_client_connect(s->export);
-    qio_channel_set_blocking(QIO_CHANNEL(sioc), false, NULL);
-    qio_channel_attach_aio_context(QIO_CHANNEL(sioc), aio_context);
+    trace_nbd_client_connect(export);
+    qio_channel_set_blocking(QIO_CHANNEL(sioc), true, NULL);

    s->info.request_sizes = true;
    s->info.structured_reply = true;
    s->info.base_allocation = true;
-    s->info.x_dirty_bitmap = g_strdup(s->x_dirty_bitmap);
-    s->info.name = g_strdup(s->export ?: "");
-    ret = nbd_receive_negotiate(aio_context, QIO_CHANNEL(sioc), s->tlscreds,
-                                s->hostname, &s->ioc, &s->info, errp);
+    s->info.x_dirty_bitmap = g_strdup(x_dirty_bitmap);
+    s->info.name = g_strdup(export ?: "");
+    ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), tlscreds, hostname,
+                                &s->ioc, &s->info, errp);
    g_free(s->info.x_dirty_bitmap);
    g_free(s->info.name);
    if (ret < 0) {
        object_unref(OBJECT(sioc));
        return ret;
    }
-    if (s->x_dirty_bitmap && !s->info.base_allocation) {
+    if (x_dirty_bitmap && !s->info.base_allocation) {
        error_setg(errp, "requested x-dirty-bitmap %s not found",
-                   s->x_dirty_bitmap);
+                   x_dirty_bitmap);
        ret = -EINVAL;
        goto fail;
    }
@@ -1454,9 +1222,6 @@ static int nbd_client_connect(BlockDriverState *bs, Error **errp)
    }
    if (s->info.flags & NBD_FLAG_SEND_WRITE_ZEROES) {
        bs->supported_zero_flags |= BDRV_REQ_MAY_UNMAP;
-        if (s->info.flags & NBD_FLAG_SEND_FAST_ZERO) {
-            bs->supported_zero_flags |= BDRV_REQ_NO_FALLBACK;
-        }
    }

    s->sioc = sioc;
@@ -1466,14 +1231,24 @@ static int nbd_client_connect(BlockDriverState *bs, Error **errp)
        object_ref(OBJECT(s->ioc));
    }

-    trace_nbd_client_connect_success(s->export);
+    /*
+     * Now that we're connected, set the socket to be non-blocking and
+     * kick the reply mechanism.
+     */
+    qio_channel_set_blocking(QIO_CHANNEL(sioc), false, NULL);
+    s->connection_co = qemu_coroutine_create(nbd_connection_entry, s);
+    bdrv_inc_in_flight(bs);
+    nbd_client_attach_aio_context(bs, bdrv_get_aio_context(bs));
+
+    trace_nbd_client_connect_success(export);

    return 0;

 fail:
    /*
-     * We have connected, but must fail for other reasons.
-     * Send NBD_CMD_DISC as a courtesy to the server.
+     * We have connected, but must fail for other reasons. The
+     * connection is still blocking; send NBD_CMD_DISC as a courtesy
+     * to the server.
     */
    {
        NBDRequest request = { .type = NBD_CMD_DISC };
@@ -1486,9 +1261,23 @@ static int nbd_client_connect(BlockDriverState *bs, Error **errp)
    }
 }

-/*
- * Parse nbd_open options
- */
+static int nbd_client_init(BlockDriverState *bs,
+                           SocketAddress *saddr,
+                           const char *export,
+                           QCryptoTLSCreds *tlscreds,
+                           const char *hostname,
+                           const char *x_dirty_bitmap,
+                           Error **errp)
+{
+    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
+
+    s->bs = bs;
+    qemu_co_mutex_init(&s->send_mutex);
+    qemu_co_queue_init(&s->free_sema);
+
+    return nbd_client_connect(bs, saddr, export, tlscreds, hostname,
+                              x_dirty_bitmap, errp);
+}

 static int nbd_parse_uri(const char *filename, QDict *options)
 {
@@ -1592,7 +1381,7 @@ static bool nbd_has_filename_options_conflict(QDict *options, Error **errp)
 static void nbd_parse_filename(const char *filename, QDict *options,
                               Error **errp)
 {
-    g_autofree char *file = NULL;
+    char *file;
    char *export_name;
    const char *host_spec;
    const char *unixpath;
@@ -1614,7 +1403,7 @@ static void nbd_parse_filename(const char *filename, QDict *options,
    export_name = strstr(file, EN_OPTSTR);
    if (export_name) {
        if (export_name[strlen(EN_OPTSTR)] == 0) {
-            return;
+            goto out;
        }
        export_name[0] = 0; /* truncate 'file' */
        export_name += strlen(EN_OPTSTR);
@@ -1625,11 +1414,11 @@ static void nbd_parse_filename(const char *filename, QDict *options,
    /* extract the host_spec - fail if it's not nbd:... */
    if (!strstart(file, "nbd:", &host_spec)) {
        error_setg(errp, "File name string for NBD must start with 'nbd:'");
-        return;
+        goto out;
    }

    if (!*host_spec) {
-        return;
+        goto out;
    }

    /* are we a UNIX or TCP socket? */
@@ -1649,6 +1438,9 @@ static void nbd_parse_filename(const char *filename, QDict *options,
    out_inet:
        qapi_free_InetSocketAddress(addr);
    }
+
+out:
+    g_free(file);
 }

 static bool nbd_process_legacy_socket_options(QDict *output_options,
@@ -1790,27 +1582,18 @@ static QemuOptsList nbd_runtime_opts = {
            .help = "experimental: expose named dirty bitmap in place of "
                    "block status",
        },
-        {
-            .name = "reconnect-delay",
-            .type = QEMU_OPT_NUMBER,
-            .help = "On an unexpected disconnect, the nbd client tries to "
-                    "connect again until succeeding or encountering a serious "
-                    "error.  During the first @reconnect-delay seconds, all "
-                    "requests are paused and will be rerun on a successful "
-                    "reconnect. After that time, any delayed requests and all "
-                    "future requests before a successful reconnect will "
-                    "immediately fail. Default 0",
-        },
        { /* end of list */ }
    },
 };

-static int nbd_process_options(BlockDriverState *bs, QDict *options,
-                               Error **errp)
+static int nbd_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
 {
    BDRVNBDState *s = bs->opaque;
-    QemuOpts *opts;
+    QemuOpts *opts = NULL;
    Error *local_err = NULL;
+    QCryptoTLSCreds *tlscreds = NULL;
+    const char *hostname = NULL;
    int ret = -EINVAL;

    opts = qemu_opts_create(&nbd_runtime_opts, NULL, 0, &error_abort);
@@ -1835,8 +1618,8 @@ static int nbd_process_options(BlockDriverState *bs, QDict *options,

    s->tlscredsid = g_strdup(qemu_opt_get(opts, "tls-creds"));
    if (s->tlscredsid) {
-        s->tlscreds = nbd_get_tls_creds(s->tlscredsid, errp);
-        if (!s->tlscreds) {
+        tlscreds = nbd_get_tls_creds(s->tlscredsid, errp);
+        if (!tlscreds) {
            goto error;
        }

@@ -1845,17 +1628,18 @@ static int nbd_process_options(BlockDriverState *bs, QDict *options,
            error_setg(errp, "TLS only supported over IP sockets");
            goto error;
        }
-        s->hostname = s->saddr->u.inet.host;
+        hostname = s->saddr->u.inet.host;
    }

-    s->x_dirty_bitmap = g_strdup(qemu_opt_get(opts, "x-dirty-bitmap"));
-    s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 0);
-
-    ret = 0;
+    /* NBD handshake */
+    ret = nbd_client_init(bs, s->saddr, s->export, tlscreds, hostname,
+                          qemu_opt_get(opts, "x-dirty-bitmap"), errp);

 error:
+    if (tlscreds) {
+        object_unref(OBJECT(tlscreds));
+    }
    if (ret < 0) {
-        object_unref(OBJECT(s->tlscreds));
        qapi_free_SocketAddress(s->saddr);
        g_free(s->export);
        g_free(s->tlscredsid);
@@ -1864,35 +1648,6 @@ static int nbd_process_options(BlockDriverState *bs, QDict *options,
    return ret;
 }

-static int nbd_open(BlockDriverState *bs, QDict *options, int flags,
-                    Error **errp)
-{
-    int ret;
-    BDRVNBDState *s = (BDRVNBDState *)bs->opaque;
-
-    ret = nbd_process_options(bs, options, errp);
-    if (ret < 0) {
-        return ret;
-    }
-
-    s->bs = bs;
-    qemu_co_mutex_init(&s->send_mutex);
-    qemu_co_queue_init(&s->free_sema);
-
-    ret = nbd_client_connect(bs, errp);
-    if (ret < 0) {
-        return ret;
-    }
-    /* successfully connected */
-    s->state = NBD_CLIENT_CONNECTED;
-
-    s->connection_co = qemu_coroutine_create(nbd_connection_entry, s);
-    bdrv_inc_in_flight(bs);
-    aio_co_schedule(bdrv_get_aio_context(bs), s->connection_co);
-
-    return 0;
-}
-
 static int nbd_co_flush(BlockDriverState *bs)
 {
    return nbd_client_co_flush(bs);
@@ -1938,11 +1693,9 @@ static void nbd_close(BlockDriverState *bs)

    nbd_client_close(bs);

-    object_unref(OBJECT(s->tlscreds));
    qapi_free_SocketAddress(s->saddr);
    g_free(s->export);
    g_free(s->tlscredsid);
-    g_free(s->x_dirty_bitmap);
 }

 static int64_t nbd_getlength(BlockDriverState *bs)
@@ -2009,7 +1762,6 @@ static BlockDriver bdrv_nbd = {
    .instance_size              = sizeof(BDRVNBDState),
    .bdrv_parse_filename        = nbd_parse_filename,
    .bdrv_file_open             = nbd_open,
-    .bdrv_reopen_prepare        = nbd_client_reopen_prepare,
    .bdrv_co_preadv             = nbd_client_co_preadv,
    .bdrv_co_pwritev            = nbd_client_co_pwritev,
    .bdrv_co_pwrite_zeroes      = nbd_client_co_pwrite_zeroes,
@@ -2020,8 +1772,6 @@ static BlockDriver bdrv_nbd = {
    .bdrv_getlength             = nbd_getlength,
    .bdrv_detach_aio_context    = nbd_client_detach_aio_context,
    .bdrv_attach_aio_context    = nbd_client_attach_aio_context,
-    .bdrv_co_drain_begin        = nbd_client_co_drain_begin,
-    .bdrv_co_drain_end          = nbd_client_co_drain_end,
    .bdrv_refresh_filename      = nbd_refresh_filename,
    .bdrv_co_block_status       = nbd_client_co_block_status,
    .bdrv_dirname               = nbd_dirname,
@@ -2034,7 +1784,6 @@ static BlockDriver bdrv_nbd_tcp = {
    .instance_size              = sizeof(BDRVNBDState),
    .bdrv_parse_filename        = nbd_parse_filename,
    .bdrv_file_open             = nbd_open,
-    .bdrv_reopen_prepare        = nbd_client_reopen_prepare,
    .bdrv_co_preadv             = nbd_client_co_preadv,
    .bdrv_co_pwritev            = nbd_client_co_pwritev,
    .bdrv_co_pwrite_zeroes      = nbd_client_co_pwrite_zeroes,
@@ -2045,8 +1794,6 @@ static BlockDriver bdrv_nbd_tcp = {
    .bdrv_getlength             = nbd_getlength,
    .bdrv_detach_aio_context    = nbd_client_detach_aio_context,
    .bdrv_attach_aio_context    = nbd_client_attach_aio_context,
-    .bdrv_co_drain_begin        = nbd_client_co_drain_begin,
-    .bdrv_co_drain_end          = nbd_client_co_drain_end,
    .bdrv_refresh_filename      = nbd_refresh_filename,
    .bdrv_co_block_status       = nbd_client_co_block_status,
    .bdrv_dirname               = nbd_dirname,
@@ -2059,7 +1806,6 @@ static BlockDriver bdrv_nbd_unix = {
    .instance_size              = sizeof(BDRVNBDState),
    .bdrv_parse_filename        = nbd_parse_filename,
    .bdrv_file_open             = nbd_open,
-    .bdrv_reopen_prepare        = nbd_client_reopen_prepare,
    .bdrv_co_preadv             = nbd_client_co_preadv,
    .bdrv_co_pwritev            = nbd_client_co_pwritev,
    .bdrv_co_pwrite_zeroes      = nbd_client_co_pwrite_zeroes,
@@ -2070,8 +1816,6 @@ static BlockDriver bdrv_nbd_unix = {
    .bdrv_getlength             = nbd_getlength,
    .bdrv_detach_aio_context    = nbd_client_detach_aio_context,
    .bdrv_attach_aio_context    = nbd_client_attach_aio_context,
-    .bdrv_co_drain_begin        = nbd_client_co_drain_begin,
-    .bdrv_co_drain_end          = nbd_client_co_drain_end,
    .bdrv_refresh_filename      = nbd_refresh_filename,
    .bdrv_co_block_status       = nbd_client_co_block_status,
    .bdrv_dirname               = nbd_dirname,
@@ -32,13 +32,11 @@
 #include "block/qdict.h"
 #include "trace.h"
 #include "qemu/iov.h"
-#include "qemu/main-loop.h"
 #include "qemu/module.h"
 #include "qemu/option.h"
 #include "qemu/uri.h"
 #include "qemu/cutils.h"
 #include "sysemu/sysemu.h"
-#include "sysemu/replay.h"
 #include "qapi/qapi-visit-block-core.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
@@ -259,8 +257,8 @@ nfs_co_generic_cb(int ret, struct nfs_context *nfs, void *data,
    if (task->ret < 0) {
        error_report("NFS Error: %s", nfs_get_error(nfs));
    }
-    replay_bh_schedule_oneshot_event(task->client->aio_context,
-                                     nfs_co_generic_bh_cb, task);
+    aio_bh_schedule_oneshot(task->client->aio_context,
+                            nfs_co_generic_bh_cb, task);
 }

 static int coroutine_fn nfs_co_preadv(BlockDriverState *bs, uint64_t offset,
@@ -400,9 +398,6 @@ static void nfs_client_close(NFSClient *client)
            nfs_close(client->context, client->fh);
            client->fh = NULL;
        }
-#ifdef LIBNFS_FEATURE_UMOUNT
-        nfs_umount(client->context);
-#endif
        nfs_destroy_context(client->context);
        client->context = NULL;
    }
@@ -752,7 +747,7 @@ static int64_t nfs_get_allocated_file_size(BlockDriverState *bs)
 }

 static int coroutine_fn
-nfs_file_co_truncate(BlockDriverState *bs, int64_t offset, bool exact,
+nfs_file_co_truncate(BlockDriverState *bs, int64_t offset,
                     PreallocMode prealloc, Error **errp)
 {
    NFSClient *client = bs->opaque;
@@ -870,7 +865,6 @@ static BlockDriver bdrv_nfs = {
    .create_opts                    = &nfs_create_opts,

    .bdrv_has_zero_init             = nfs_has_zero_init,
-    .bdrv_has_zero_init_truncate    = nfs_has_zero_init,
    .bdrv_get_allocated_file_size   = nfs_get_allocated_file_size,
    .bdrv_co_truncate               = nfs_file_co_truncate,

@@ -17,7 +17,6 @@
 #include "qemu/module.h"
 #include "qemu/option.h"
 #include "block/block_int.h"
-#include "sysemu/replay.h"

 #define NULL_OPT_LATENCY "latency-ns"
 #define NULL_OPT_ZEROES  "read-zeroes"
@@ -180,8 +179,7 @@ static inline BlockAIOCB *null_aio_common(BlockDriverState *bs,
        timer_mod_ns(&acb->timer,
                     qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + s->latency_ns);
    } else {
-        replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs),
-                                         null_bh_cb, acb);
+        aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), null_bh_cb, acb);
    }
    return &acb->common;
 }
@@ -17,13 +17,11 @@
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
 #include "qemu/error-report.h"
-#include "qemu/main-loop.h"
 #include "qemu/module.h"
 #include "qemu/cutils.h"
 #include "qemu/option.h"
 #include "qemu/vfio-helpers.h"
 #include "block/block_int.h"
-#include "sysemu/replay.h"
 #include "trace.h"

 #include "block/nvme.h"
@@ -112,9 +110,6 @@ typedef struct {
    uint64_t max_transfer;
    bool plugged;

-    bool supports_write_zeroes;
-    bool supports_discard;
-
    CoMutex dma_map_lock;
    CoQueue dma_flush_queue;

@@ -355,8 +350,7 @@ static bool nvme_process_completion(BDRVNVMeState *s, NVMeQueuePair *q)
        smp_mb_release();
        *q->cq.doorbell = cpu_to_le32(q->cq.head);
        if (!qemu_co_queue_empty(&q->free_req_queue)) {
-            replay_bh_schedule_oneshot_event(s->aio_context,
-                                             nvme_free_req_queue_cb, q);
+            aio_bh_schedule_oneshot(s->aio_context, nvme_free_req_queue_cb, q);
        }
    }
    q->busy = false;
@@ -426,7 +420,6 @@ static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
    NvmeIdNs *idns;
    NvmeLBAF *lbaf;
    uint8_t *resp;
-    uint16_t oncs;
    int r;
    uint64_t iova;
    NvmeCmd cmd = {
@@ -464,10 +457,6 @@ static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
    s->max_transfer = MIN_NON_ZERO(s->max_transfer,
                          s->page_size / sizeof(uint64_t) * s->page_size);

-    oncs = le16_to_cpu(idctrl->oncs);
-    s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROS);
-    s->supports_discard = !!(oncs & NVME_ONCS_DSM);
-
    memset(resp, 0, 4096);

    cmd.cdw10 = 0;
@@ -480,12 +469,6 @@ static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
    s->nsze = le64_to_cpu(idns->nsze);
    lbaf = &idns->lbaf[NVME_ID_NS_FLBAS_INDEX(idns->flbas)];

-    if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(idns->dlfeat) &&
-            NVME_ID_NS_DLFEAT_READ_BEHAVIOR(idns->dlfeat) ==
-                    NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) {
-        bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP;
-    }
-
    if (lbaf->ms) {
        error_setg(errp, "Namespaces with metadata are not yet supported");
        goto out;
@@ -780,8 +763,6 @@ static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
    int ret;
    BDRVNVMeState *s = bs->opaque;

-    bs->supported_write_flags = BDRV_REQ_FUA;
-
    opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
    qemu_opts_absorb_qdict(opts, options, &error_abort);
    device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
@@ -810,6 +791,7 @@ static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
            goto fail;
        }
    }
+    bs->supported_write_flags = BDRV_REQ_FUA;
    return 0;
 fail:
    nvme_close(bs);
@@ -952,7 +934,7 @@ static void nvme_rw_cb(void *opaque, int ret)
        /* The rw coroutine hasn't yielded, don't try to enter. */
        return;
    }
-    replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data);
+    aio_bh_schedule_oneshot(data->ctx, nvme_rw_cb_bh, data);
 }

 static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
@@ -1103,140 +1085,6 @@ static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
 }


-static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs,
-                                              int64_t offset,
-                                              int bytes,
-                                              BdrvRequestFlags flags)
-{
-    BDRVNVMeState *s = bs->opaque;
-    NVMeQueuePair *ioq = s->queues[1];
-    NVMeRequest *req;
-
-    uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF;
-
-    if (!s->supports_write_zeroes) {
-        return -ENOTSUP;
-    }
-
-    NvmeCmd cmd = {
-        .opcode = NVME_CMD_WRITE_ZEROS,
-        .nsid = cpu_to_le32(s->nsid),
-        .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
-        .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
-    };
-
-    NVMeCoData data = {
-        .ctx = bdrv_get_aio_context(bs),
-        .ret = -EINPROGRESS,
-    };
-
-    if (flags & BDRV_REQ_MAY_UNMAP) {
-        cdw12 |= (1 << 25);
-    }
-
-    if (flags & BDRV_REQ_FUA) {
-        cdw12 |= (1 << 30);
-    }
-
-    cmd.cdw12 = cpu_to_le32(cdw12);
-
-    trace_nvme_write_zeroes(s, offset, bytes, flags);
-    assert(s->nr_queues > 1);
-    req = nvme_get_free_req(ioq);
-    assert(req);
-
-    nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
-
-    data.co = qemu_coroutine_self();
-    while (data.ret == -EINPROGRESS) {
-        qemu_coroutine_yield();
-    }
-
-    trace_nvme_rw_done(s, true, offset, bytes, data.ret);
-    return data.ret;
-}
-
-
-static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs,
-                                         int64_t offset,
-                                         int bytes)
-{
-    BDRVNVMeState *s = bs->opaque;
-    NVMeQueuePair *ioq = s->queues[1];
-    NVMeRequest *req;
-    NvmeDsmRange *buf;
-    QEMUIOVector local_qiov;
-    int ret;
-
-    NvmeCmd cmd = {
-        .opcode = NVME_CMD_DSM,
-        .nsid = cpu_to_le32(s->nsid),
-        .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/
-        .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/
-    };
-
-    NVMeCoData data = {
-        .ctx = bdrv_get_aio_context(bs),
-        .ret = -EINPROGRESS,
-    };
-
-    if (!s->supports_discard) {
-        return -ENOTSUP;
-    }
-
-    assert(s->nr_queues > 1);
-
-    buf = qemu_try_blockalign0(bs, s->page_size);
-    if (!buf) {
-        return -ENOMEM;
-    }
-
-    buf->nlb = cpu_to_le32(bytes >> s->blkshift);
-    buf->slba = cpu_to_le64(offset >> s->blkshift);
-    buf->cattr = 0;
-
-    qemu_iovec_init(&local_qiov, 1);
-    qemu_iovec_add(&local_qiov, buf, 4096);
-
-    req = nvme_get_free_req(ioq);
-    assert(req);
-
-    qemu_co_mutex_lock(&s->dma_map_lock);
-    ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov);
-    qemu_co_mutex_unlock(&s->dma_map_lock);
-
-    if (ret) {
-        req->busy = false;
-        goto out;
-    }
-
-    trace_nvme_dsm(s, offset, bytes);
-
-    nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
-
-    data.co = qemu_coroutine_self();
-    while (data.ret == -EINPROGRESS) {
-        qemu_coroutine_yield();
-    }
-
-    qemu_co_mutex_lock(&s->dma_map_lock);
-    ret = nvme_cmd_unmap_qiov(bs, &local_qiov);
-    qemu_co_mutex_unlock(&s->dma_map_lock);
-
-    if (ret) {
-        goto out;
-    }
-
-    ret = data.ret;
-    trace_nvme_dsm_done(s, offset, bytes, ret);
-out:
-    qemu_iovec_destroy(&local_qiov);
-    qemu_vfree(buf);
-    return ret;
-
-}
-
-
 static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
                               BlockReopenQueue *queue, Error **errp)
 {
@@ -1341,10 +1189,6 @@ static BlockDriver bdrv_nvme = {

    .bdrv_co_preadv           = nvme_co_preadv,
    .bdrv_co_pwritev          = nvme_co_pwritev,
-
-    .bdrv_co_pwrite_zeroes    = nvme_co_pwrite_zeroes,
-    .bdrv_co_pdiscard         = nvme_co_pdiscard,
-
    .bdrv_co_flush_to_disk    = nvme_co_flush,
    .bdrv_reopen_prepare      = nvme_reopen_prepare,

@@ -203,7 +203,7 @@ static int64_t allocate_clusters(BlockDriverState *bs, int64_t sector_num,
        } else {
            ret = bdrv_truncate(bs->file,
                                (s->data_end + space) << BDRV_SECTOR_BITS,
-                                false, PREALLOC_MODE_OFF, NULL);
+                                PREALLOC_MODE_OFF, NULL);
        }
        if (ret < 0) {
            return ret;
@@ -487,12 +487,7 @@ static int coroutine_fn parallels_co_check(BlockDriverState *bs,
        res->leaks += count;
        if (fix & BDRV_FIX_LEAKS) {
            Error *local_err = NULL;
-
-            /*
-             * In order to really repair the image, we must shrink it.
-             * That means we have to pass exact=true.
-             */
-            ret = bdrv_truncate(bs->file, res->image_end_offset, true,
+            ret = bdrv_truncate(bs->file, res->image_end_offset,
                                PREALLOC_MODE_OFF, &local_err);
            if (ret < 0) {
                error_report_err(local_err);
@@ -568,6 +563,11 @@ static int coroutine_fn parallels_co_create(BlockdevCreateOptions* opts,
    blk_set_allow_write_beyond_eof(blk, true);

    /* Create image format */
+    ret = blk_truncate(blk, 0, PREALLOC_MODE_OFF, errp);
+    if (ret < 0) {
+        goto out;
+    }
+
    bat_entries = DIV_ROUND_UP(total_size, cl_size);
    bat_sectors = DIV_ROUND_UP(bat_entry_off(bat_entries), cl_size);
    bat_sectors = (bat_sectors *  cl_size) >> BDRV_SECTOR_BITS;
@@ -835,7 +835,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
        goto fail_options;
    }

-    if (!bdrv_has_zero_init_truncate(bs->file->bs)) {
+    if (!bdrv_has_zero_init(bs->file->bs)) {
        s->prealloc_mode = PRL_PREALLOC_MODE_FALLOCATE;
    }

@@ -847,7 +847,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
        }
    }

-    s->bat_dirty_block = 4 * qemu_real_host_page_size;
+    s->bat_dirty_block = 4 * getpagesize();
    s->bat_dirty_bmap =
        bitmap_new(DIV_ROUND_UP(s->header_size, s->bat_dirty_block));

@@ -885,9 +885,7 @@ static void parallels_close(BlockDriverState *bs)
    if ((bs->open_flags & BDRV_O_RDWR) && !(bs->open_flags & BDRV_O_INACTIVE)) {
        s->header->inuse = 0;
        parallels_update_header(bs);
-
-        /* errors are ignored, so we might as well pass exact=true */
-        bdrv_truncate(bs->file, s->data_end << BDRV_SECTOR_BITS, true,
+        bdrv_truncate(bs->file, s->data_end << BDRV_SECTOR_BITS,
                      PREALLOC_MODE_OFF, NULL);
    }

@@ -23,7 +23,7 @@
 */

 #include "qemu/osdep.h"
-#include "qemu/cutils.h"
+#include "qemu-common.h"
 #include "block/qapi.h"
 #include "block/block_int.h"
 #include "block/throttle-groups.h"
@@ -79,11 +79,6 @@ BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
        info->backing_file = g_strdup(bs->backing_file);
    }

-    if (!QLIST_EMPTY(&bs->dirty_bitmaps)) {
-        info->has_dirty_bitmaps = true;
-        info->dirty_bitmaps = bdrv_query_dirty_bitmaps(bs);
-    }
-
    info->detect_zeroes = bs->detect_zeroes;

    if (blk && blk_get_public(blk)->throttle_group_member.throttle_state) {
@@ -440,30 +435,24 @@ static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk)

    ds->rd_bytes = stats->nr_bytes[BLOCK_ACCT_READ];
    ds->wr_bytes = stats->nr_bytes[BLOCK_ACCT_WRITE];
-    ds->unmap_bytes = stats->nr_bytes[BLOCK_ACCT_UNMAP];
    ds->rd_operations = stats->nr_ops[BLOCK_ACCT_READ];
    ds->wr_operations = stats->nr_ops[BLOCK_ACCT_WRITE];
-    ds->unmap_operations = stats->nr_ops[BLOCK_ACCT_UNMAP];

    ds->failed_rd_operations = stats->failed_ops[BLOCK_ACCT_READ];
    ds->failed_wr_operations = stats->failed_ops[BLOCK_ACCT_WRITE];
    ds->failed_flush_operations = stats->failed_ops[BLOCK_ACCT_FLUSH];
-    ds->failed_unmap_operations = stats->failed_ops[BLOCK_ACCT_UNMAP];

    ds->invalid_rd_operations = stats->invalid_ops[BLOCK_ACCT_READ];
    ds->invalid_wr_operations = stats->invalid_ops[BLOCK_ACCT_WRITE];
    ds->invalid_flush_operations =
        stats->invalid_ops[BLOCK_ACCT_FLUSH];
-    ds->invalid_unmap_operations = stats->invalid_ops[BLOCK_ACCT_UNMAP];

    ds->rd_merged = stats->merged[BLOCK_ACCT_READ];
    ds->wr_merged = stats->merged[BLOCK_ACCT_WRITE];
-    ds->unmap_merged = stats->merged[BLOCK_ACCT_UNMAP];
    ds->flush_operations = stats->nr_ops[BLOCK_ACCT_FLUSH];
    ds->wr_total_time_ns = stats->total_time_ns[BLOCK_ACCT_WRITE];
    ds->rd_total_time_ns = stats->total_time_ns[BLOCK_ACCT_READ];
    ds->flush_total_time_ns = stats->total_time_ns[BLOCK_ACCT_FLUSH];
-    ds->unmap_total_time_ns = stats->total_time_ns[BLOCK_ACCT_UNMAP];

    ds->has_idle_time_ns = stats->last_access_time_ns > 0;
    if (ds->has_idle_time_ns) {
@@ -543,11 +532,6 @@ static BlockStats *bdrv_query_bds_stats(BlockDriverState *bs,

    s->stats->wr_highest_offset = stat64_get(&bs->wr_highest_offset);

-    s->driver_specific = bdrv_get_specific_stats(bs);
-    if (s->driver_specific) {
-        s->has_driver_specific = true;
-    }
-
    if (bs->file) {
        s->has_parent = true;
        s->parent = bdrv_query_bds_stats(bs->file->bs, blk_level);
@@ -480,7 +480,7 @@ static int get_cluster_offset(BlockDriverState *bs,
                    return -E2BIG;
                }
                ret = bdrv_truncate(bs->file, cluster_offset + s->cluster_size,
-                                    false, PREALLOC_MODE_OFF, NULL);
+                                    PREALLOC_MODE_OFF, NULL);
                if (ret < 0) {
                    return ret;
                }
@@ -858,6 +858,11 @@ static int coroutine_fn qcow_co_create(BlockdevCreateOptions *opts,
    blk_set_allow_write_beyond_eof(qcow_blk, true);

    /* Create image format */
+    ret = blk_truncate(qcow_blk, 0, PREALLOC_MODE_OFF, errp);
+    if (ret < 0) {
+        goto exit;
+    }
+
    memset(&header, 0, sizeof(header));
    header.magic = cpu_to_be32(QCOW_MAGIC);
    header.version = cpu_to_be32(QCOW_VERSION);
@@ -1033,7 +1038,7 @@ static int qcow_make_empty(BlockDriverState *bs)
    if (bdrv_pwrite_sync(bs->file, s->l1_table_offset, s->l1_table,
            l1_length) < 0)
        return -1;
-    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length, false,
+    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length,
                        PREALLOC_MODE_OFF, NULL);
    if (ret < 0)
        return ret;
@@ -382,7 +382,7 @@ static BdrvDirtyBitmap *load_bitmap(BlockDriverState *bs,
 fail:
    g_free(bitmap_table);
    if (bitmap != NULL) {
-        bdrv_release_dirty_bitmap(bitmap);
+        bdrv_release_dirty_bitmap(bs, bitmap);
    }

    return NULL;
@@ -949,7 +949,7 @@ fail:
 static void release_dirty_bitmap_helper(gpointer bitmap,
                                        gpointer bs)
 {
-    bdrv_release_dirty_bitmap(bitmap);
+    bdrv_release_dirty_bitmap(bs, bitmap);
 }

 /* for g_slist_foreach for GSList of BdrvDirtyBitmap* elements */
@@ -1110,20 +1110,29 @@ Qcow2BitmapInfoList *qcow2_get_bitmap_info_list(BlockDriverState *bs,
    return list;
 }

-int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
+int qcow2_reopen_bitmaps_rw_hint(BlockDriverState *bs, bool *header_updated,
+                                 Error **errp)
 {
    BDRVQcow2State *s = bs->opaque;
    Qcow2BitmapList *bm_list;
    Qcow2Bitmap *bm;
    GSList *ro_dirty_bitmaps = NULL;
-    int ret = -EINVAL;
-    bool need_header_update = false;
+    int ret = 0;
+
+    if (header_updated != NULL) {
+        *header_updated = false;
+    }

    if (s->nb_bitmaps == 0) {
        /* No bitmaps - nothing to do */
        return 0;
    }

+    if (!can_write(bs)) {
+        error_setg(errp, "Can't write to the image on reopening bitmaps rw");
+        return -EINVAL;
+    }
+
    bm_list = bitmap_list_load(bs, s->bitmap_directory_offset,
                               s->bitmap_directory_size, errp);
    if (bm_list == NULL) {
@@ -1132,75 +1141,35 @@ int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)

    QSIMPLEQ_FOREACH(bm, bm_list, entry) {
        BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(bs, bm->name);
+        if (bitmap == NULL) {
+            continue;
+        }

-        if (!bitmap) {
-            error_setg(errp, "Unexpected bitmap '%s' in image '%s'",
-                       bm->name, bs->filename);
+        if (!bdrv_dirty_bitmap_readonly(bitmap)) {
+            error_setg(errp, "Bitmap %s was loaded prior to rw-reopen, but was "
+                       "not marked as readonly. This is a bug, something went "
+                       "wrong. All of the bitmaps may be corrupted", bm->name);
+            ret = -EINVAL;
            goto out;
        }

-        if (!(bm->flags & BME_FLAG_IN_USE)) {
-            if (!bdrv_dirty_bitmap_readonly(bitmap)) {
-                error_setg(errp, "Corruption: bitmap '%s' is not marked IN_USE "
-                           "in the image '%s' and not marked readonly in RAM",
-                           bm->name, bs->filename);
-                goto out;
-            }
-            if (bdrv_dirty_bitmap_inconsistent(bitmap)) {
-                error_setg(errp, "Corruption: bitmap '%s' is inconsistent but "
-                           "is not marked IN_USE in the image '%s'", bm->name,
-                           bs->filename);
-                goto out;
-            }
-
-            bm->flags |= BME_FLAG_IN_USE;
-            need_header_update = true;
-        } else {
-            /*
-             * What if flags already has BME_FLAG_IN_USE ?
-             *
-             * 1. if we are reopening RW -> RW it's OK, of course.
-             * 2. if we are reopening RO -> RW:
-             *   2.1 if @bitmap is inconsistent, it's OK. It means that it was
-             *       inconsistent (IN_USE) when we loaded it
-             *   2.2 if @bitmap is not inconsistent. This seems to be impossible
-             *       and implies third party interaction. Let's error-out for
-             *       safety.
-             */
-            if (bdrv_dirty_bitmap_readonly(bitmap) &&
-                !bdrv_dirty_bitmap_inconsistent(bitmap))
-            {
-                error_setg(errp, "Corruption: bitmap '%s' is marked IN_USE "
-                           "in the image '%s' but it is readonly and "
-                           "consistent in RAM",
-                           bm->name, bs->filename);
-                goto out;
-            }
-        }
-
-        if (bdrv_dirty_bitmap_readonly(bitmap)) {
-            ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
-        }
+        bm->flags |= BME_FLAG_IN_USE;
+        ro_dirty_bitmaps = g_slist_append(ro_dirty_bitmaps, bitmap);
    }

-    if (need_header_update) {
-        if (!can_write(bs->file->bs) || !(bs->file->perm & BLK_PERM_WRITE)) {
-            error_setg(errp, "Failed to reopen bitmaps rw: no write access "
-                       "the protocol file");
-            goto out;
-        }
-
+    if (ro_dirty_bitmaps != NULL) {
        /* in_use flags must be updated */
        ret = update_ext_header_and_dir_in_place(bs, bm_list);
        if (ret < 0) {
-            error_setg_errno(errp, -ret, "Cannot update bitmap directory");
+            error_setg_errno(errp, -ret, "Can't update bitmap directory");
            goto out;
        }
+        if (header_updated != NULL) {
+            *header_updated = true;
+        }
+        g_slist_foreach(ro_dirty_bitmaps, set_readonly_helper, false);
    }

-    g_slist_foreach(ro_dirty_bitmaps, set_readonly_helper, false);
-    ret = 0;
-
 out:
    g_slist_free(ro_dirty_bitmaps);
    bitmap_list_free(bm_list);
@@ -1208,6 +1177,11 @@ out:
    return ret;
 }

+int qcow2_reopen_bitmaps_rw(BlockDriverState *bs, Error **errp)
+{
+    return qcow2_reopen_bitmaps_rw_hint(bs, NULL, errp);
+}
+
 /* Checks to see if it's safe to resize bitmaps */
 int qcow2_truncate_bitmaps_check(BlockDriverState *bs, Error **errp)
 {
@@ -1438,34 +1412,30 @@ static Qcow2Bitmap *find_bitmap_by_name(Qcow2BitmapList *bm_list,
    return NULL;
 }

-int coroutine_fn qcow2_co_remove_persistent_dirty_bitmap(BlockDriverState *bs,
-                                                         const char *name,
-                                                         Error **errp)
+void qcow2_remove_persistent_dirty_bitmap(BlockDriverState *bs,
+                                          const char *name,
+                                          Error **errp)
 {
    int ret;
    BDRVQcow2State *s = bs->opaque;
-    Qcow2Bitmap *bm = NULL;
+    Qcow2Bitmap *bm;
    Qcow2BitmapList *bm_list;

    if (s->nb_bitmaps == 0) {
        /* Absence of the bitmap is not an error: see explanation above
         * bdrv_remove_persistent_dirty_bitmap() definition. */
-        return 0;
+        return;
    }

-    qemu_co_mutex_lock(&s->lock);
-
    bm_list = bitmap_list_load(bs, s->bitmap_directory_offset,
                               s->bitmap_directory_size, errp);
    if (bm_list == NULL) {
-        ret = -EIO;
-        goto out;
+        return;
    }

    bm = find_bitmap_by_name(bm_list, name);
    if (bm == NULL) {
-        ret = -EINVAL;
-        goto out;
+        goto fail;
    }

    QSIMPLEQ_REMOVE(bm_list, bm, Qcow2Bitmap, entry);
@@ -1473,46 +1443,17 @@ int coroutine_fn qcow2_co_remove_persistent_dirty_bitmap(BlockDriverState *bs,
    ret = update_ext_header_and_dir(bs, bm_list);
    if (ret < 0) {
        error_setg_errno(errp, -ret, "Failed to update bitmap extension");
-        goto out;
+        goto fail;
    }

    free_bitmap_clusters(bs, &bm->table);

-out:
-    qemu_co_mutex_unlock(&s->lock);
-
+fail:
    bitmap_free(bm);
    bitmap_list_free(bm_list);
-
-    return ret;
 }

-/*
- * qcow2_store_persistent_dirty_bitmaps
- *
- * Stores persistent BdrvDirtyBitmap objects.
- *
- * @release_stored: if true, release BdrvDirtyBitmap's after storing to the
- * image. This is used in two cases, both via qcow2_inactivate:
- * 1. bdrv_close: It's correct to remove bitmaps on close.
- * 2. migration: If bitmaps are migrated through migration channel via
- *    'dirty-bitmaps' migration capability they are not handled by this code.
- *    Otherwise, it's OK to drop BdrvDirtyBitmap's and reload them on
- *    invalidation.
- *
- * Anyway, it's correct to remove BdrvDirtyBitmap's on inactivation, as
- * inactivation means that we lose control on disk, and therefore on bitmaps,
- * we should sync them and do not touch more.
- *
- * Contrariwise, we don't want to release any bitmaps on just reopen-to-ro,
- * when we need to store them, as image is still under our control, and it's
- * good to keep all the bitmaps in read-only mode. Moreover, keeping them
- * read-only is correct because this is what would happen if we opened the node
- * readonly to begin with, and whether we opened directly or reopened to that
- * state shouldn't matter for the state we get afterward.
- */
-void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
-                                          bool release_stored, Error **errp)
+void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs, Error **errp)
 {
    BdrvDirtyBitmap *bitmap;
    BDRVQcow2State *s = bs->opaque;
@@ -1523,7 +1464,16 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
    Qcow2Bitmap *bm;
    QSIMPLEQ_HEAD(, Qcow2BitmapTable) drop_tables;
    Qcow2BitmapTable *tb, *tb_next;
-    bool need_write = false;
+
+    if (!bdrv_has_changed_persistent_bitmaps(bs)) {
+        /* nothing to do */
+        return;
+    }
+
+    if (!can_write(bs)) {
+        error_setg(errp, "No write access");
+        return;
+    }

    QSIMPLEQ_INIT(&drop_tables);

@@ -1538,7 +1488,9 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
    }

    /* check constraints and names */
-    FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
+    for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
+         bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
+    {
        const char *name = bdrv_dirty_bitmap_name(bitmap);
        uint32_t granularity = bdrv_dirty_bitmap_granularity(bitmap);
        Qcow2Bitmap *bm;
@@ -1549,8 +1501,6 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
            continue;
        }

-        need_write = true;
-
        if (check_constraints_on_bitmap(bs, name, granularity, errp) < 0) {
            error_prepend(errp, "Bitmap '%s' doesn't satisfy the constraints: ",
                          name);
@@ -1589,15 +1539,6 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
        bm->dirty_bitmap = bitmap;
    }

-    if (!need_write) {
-        goto success;
-    }
-
-    if (!can_write(bs)) {
-        error_setg(errp, "No write access");
-        goto fail;
-    }
-
    /* allocate clusters and store bitmaps */
    QSIMPLEQ_FOREACH(bm, bm_list, entry) {
        if (bm->dirty_bitmap == NULL) {
@@ -1623,17 +1564,22 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
        g_free(tb);
    }

-    if (release_stored) {
-        QSIMPLEQ_FOREACH(bm, bm_list, entry) {
-            if (bm->dirty_bitmap == NULL) {
-                continue;
-            }
-
-            bdrv_release_dirty_bitmap(bm->dirty_bitmap);
+    QSIMPLEQ_FOREACH(bm, bm_list, entry) {
+        /* For safety, we remove bitmap after storing.
+         * We may be here in two cases:
+         * 1. bdrv_close. It's ok to drop bitmap.
+         * 2. inactivation. It means migration without 'dirty-bitmaps'
+         *    capability, so bitmaps are not marked with
+         *    BdrvDirtyBitmap.migration flags. It's not bad to drop them too,
+         *    and reload on invalidation.
+         */
+        if (bm->dirty_bitmap == NULL) {
+            continue;
        }
+
+        bdrv_release_dirty_bitmap(bs, bm->dirty_bitmap);
    }

-success:
    bitmap_list_free(bm_list);
    return;

@@ -1658,13 +1604,15 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error **errp)
    BdrvDirtyBitmap *bitmap;
    Error *local_err = NULL;

-    qcow2_store_persistent_dirty_bitmaps(bs, false, &local_err);
+    qcow2_store_persistent_dirty_bitmaps(bs, &local_err);
    if (local_err != NULL) {
        error_propagate(errp, local_err);
        return -EINVAL;
    }

-    FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
+    for (bitmap = bdrv_dirty_bitmap_next(bs, NULL); bitmap != NULL;
+         bitmap = bdrv_dirty_bitmap_next(bs, bitmap))
+    {
        if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
            bdrv_dirty_bitmap_set_readonly(bitmap, true);
        }
@@ -1673,10 +1621,10 @@ int qcow2_reopen_bitmaps_ro(BlockDriverState *bs, Error **errp)
    return 0;
 }

-bool coroutine_fn qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
-                                                      const char *name,
-                                                      uint32_t granularity,
-                                                      Error **errp)
+bool qcow2_can_store_new_dirty_bitmap(BlockDriverState *bs,
+                                      const char *name,
+                                      uint32_t granularity,
+                                      Error **errp)
 {
    BDRVQcow2State *s = bs->opaque;
    bool found;
@@ -1713,10 +1661,8 @@ bool coroutine_fn qcow2_co_can_store_new_dirty_bitmap(BlockDriverState *bs,
        goto fail;
    }

-    qemu_co_mutex_lock(&s->lock);
    bm_list = bitmap_list_load(bs, s->bitmap_directory_offset,
                               s->bitmap_directory_size, errp);
-    qemu_co_mutex_unlock(&s->lock);
    if (bm_list == NULL) {
        goto fail;
    }
@@ -74,7 +74,7 @@ static void qcow2_cache_table_release(Qcow2Cache *c, int i, int num_tables)
 /* Using MADV_DONTNEED to discard memory is a Linux-specific feature */
 #ifdef CONFIG_LINUX
    void *t = qcow2_cache_get_table_addr(c, i);
-    int align = qemu_real_host_page_size;
+    int align = getpagesize();
    size_t mem_size = (size_t) c->table_size * num_tables;
    size_t offset = QEMU_ALIGN_UP((uintptr_t) t, align) - (uintptr_t) t;
    size_t length = QEMU_ALIGN_DOWN(mem_size - offset, align);
@@ -452,9 +452,8 @@ static int coroutine_fn do_perform_cow_read(BlockDriverState *bs,
     * interface.  This avoids double I/O throttling and request tracking,
     * which can lead to deadlock when block layer copy-on-read is enabled.
     */
-    ret = bs->drv->bdrv_co_preadv_part(bs,
-                                       src_cluster_offset + offset_in_cluster,
-                                       qiov->size, qiov, 0, 0);
+    ret = bs->drv->bdrv_co_preadv(bs, src_cluster_offset + offset_in_cluster,
+                                  qiov->size, qiov, 0);
    if (ret < 0) {
        return ret;
    }
@@ -462,6 +461,28 @@ static int coroutine_fn do_perform_cow_read(BlockDriverState *bs,
    return 0;
 }

+static bool coroutine_fn do_perform_cow_encrypt(BlockDriverState *bs,
+                                                uint64_t src_cluster_offset,
+                                                uint64_t cluster_offset,
+                                                unsigned offset_in_cluster,
+                                                uint8_t *buffer,
+                                                unsigned bytes)
+{
+    if (bytes && bs->encrypted) {
+        BDRVQcow2State *s = bs->opaque;
+        assert((offset_in_cluster & ~BDRV_SECTOR_MASK) == 0);
+        assert((bytes & ~BDRV_SECTOR_MASK) == 0);
+        assert(s->crypto);
+        if (qcow2_co_encrypt(bs,
+                start_of_cluster(s, cluster_offset + offset_in_cluster),
+                src_cluster_offset + offset_in_cluster,
+                buffer, bytes) < 0) {
+            return false;
+        }
+    }
+    return true;
+}
+
 static int coroutine_fn do_perform_cow_write(BlockDriverState *bs,
                                             uint64_t cluster_offset,
                                             unsigned offset_in_cluster,
@@ -808,6 +829,7 @@ static int perform_cow(BlockDriverState *bs, QCowL2Meta *m)
    assert(start->nb_bytes <= UINT_MAX - end->nb_bytes);
    assert(start->nb_bytes + end->nb_bytes <= UINT_MAX - data_bytes);
    assert(start->offset + start->nb_bytes <= end->offset);
+    assert(!m->data_qiov || m->data_qiov->size == data_bytes);

    if ((start->nb_bytes == 0 && end->nb_bytes == 0) || m->skip_cow) {
        return 0;
@@ -839,11 +861,7 @@ static int perform_cow(BlockDriverState *bs, QCowL2Meta *m)
    /* The part of the buffer where the end region is located */
    end_buffer = start_buffer + buffer_size - end->nb_bytes;

-    qemu_iovec_init(&qiov, 2 + (m->data_qiov ?
-                                qemu_iovec_subvec_niov(m->data_qiov,
-                                                       m->data_qiov_offset,
-                                                       data_bytes)
-                                : 0));
+    qemu_iovec_init(&qiov, 2 + (m->data_qiov ? m->data_qiov->niov : 0));

    qemu_co_mutex_unlock(&s->lock);
    /* First we read the existing data from both COW regions. We
@@ -869,19 +887,12 @@ static int perform_cow(BlockDriverState *bs, QCowL2Meta *m)

    /* Encrypt the data if necessary before writing it */
    if (bs->encrypted) {
-        ret = qcow2_co_encrypt(bs,
-                               m->alloc_offset + start->offset,
-                               m->offset + start->offset,
-                               start_buffer, start->nb_bytes);
-        if (ret < 0) {
-            goto fail;
-        }
-
-        ret = qcow2_co_encrypt(bs,
-                               m->alloc_offset + end->offset,
-                               m->offset + end->offset,
-                               end_buffer, end->nb_bytes);
-        if (ret < 0) {
+        if (!do_perform_cow_encrypt(bs, m->offset, m->alloc_offset,
+                                    start->offset, start_buffer,
+                                    start->nb_bytes) ||
+            !do_perform_cow_encrypt(bs, m->offset, m->alloc_offset,
+                                    end->offset, end_buffer, end->nb_bytes)) {
+            ret = -EIO;
            goto fail;
        }
    }
@@ -893,7 +904,7 @@ static int perform_cow(BlockDriverState *bs, QCowL2Meta *m)
        if (start->nb_bytes) {
            qemu_iovec_add(&qiov, start_buffer, start->nb_bytes);
        }
-        qemu_iovec_concat(&qiov, m->data_qiov, m->data_qiov_offset, data_bytes);
+        qemu_iovec_concat(&qiov, m->data_qiov, 0, data_bytes);
        if (end->nb_bytes) {
            qemu_iovec_add(&qiov, end_buffer, end->nb_bytes);
        }
@@ -1340,7 +1351,13 @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
    }

    entry = be64_to_cpu(l2_slice[l2_index]);
-    nb_clusters = count_cow_clusters(bs, nb_clusters, l2_slice, l2_index);
+
+    /* For the moment, overwrite compressed clusters one by one */
+    if (entry & QCOW_OFLAG_COMPRESSED) {
+        nb_clusters = 1;
+    } else {
+        nb_clusters = count_cow_clusters(bs, nb_clusters, l2_slice, l2_index);
+    }

    /* This function is only called when there were no non-COW clusters, so if
     * we can't find any unallocated or COW clusters either, something is
@@ -2016,7 +2016,7 @@ static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
                    goto resize_fail;
                }

-                ret = bdrv_truncate(bs->file, offset + s->cluster_size, false,
+                ret = bdrv_truncate(bs->file, offset + s->cluster_size,
                                    PREALLOC_MODE_OFF, &local_err);
                if (ret < 0) {
                    error_report_err(local_err);
@@ -29,64 +29,29 @@
 #include "qemu/error-report.h"
 #include "qemu/cutils.h"

-static void qcow2_free_single_snapshot(BlockDriverState *bs, int i)
-{
-    BDRVQcow2State *s = bs->opaque;
-
-    assert(i >= 0 && i < s->nb_snapshots);
-    g_free(s->snapshots[i].name);
-    g_free(s->snapshots[i].id_str);
-    g_free(s->snapshots[i].unknown_extra_data);
-    memset(&s->snapshots[i], 0, sizeof(s->snapshots[i]));
-}
-
 void qcow2_free_snapshots(BlockDriverState *bs)
 {
    BDRVQcow2State *s = bs->opaque;
    int i;

    for(i = 0; i < s->nb_snapshots; i++) {
-        qcow2_free_single_snapshot(bs, i);
+        g_free(s->snapshots[i].name);
+        g_free(s->snapshots[i].id_str);
    }
    g_free(s->snapshots);
    s->snapshots = NULL;
    s->nb_snapshots = 0;
 }

-/*
- * If @repair is true, try to repair a broken snapshot table instead
- * of just returning an error:
- *
- * - If the snapshot table was too long, set *nb_clusters_reduced to
- *   the number of snapshots removed off the end.
- *   The caller will update the on-disk nb_snapshots accordingly;
- *   this leaks clusters, but is safe.
- *   (The on-disk information must be updated before
- *   qcow2_check_refcounts(), because that function relies on
- *   s->nb_snapshots to reflect the on-disk value.)
- *
- * - If there were snapshots with too much extra metadata, increment
- *   *extra_data_dropped for each.
- *   This requires the caller to eventually rewrite the whole snapshot
- *   table, which requires cluster allocation.  Therefore, this should
- *   be done only after qcow2_check_refcounts() made sure the refcount
- *   structures are valid.
- *   (In the meantime, the image is still valid because
- *   qcow2_check_refcounts() does not do anything with snapshots'
- *   extra data.)
- */
-static int qcow2_do_read_snapshots(BlockDriverState *bs, bool repair,
-                                   int *nb_clusters_reduced,
-                                   int *extra_data_dropped,
-                                   Error **errp)
+int qcow2_read_snapshots(BlockDriverState *bs)
 {
    BDRVQcow2State *s = bs->opaque;
    QCowSnapshotHeader h;
    QCowSnapshotExtraData extra;
    QCowSnapshot *sn;
    int i, id_str_size, name_size;
-    int64_t offset, pre_sn_offset;
-    uint64_t table_length = 0;
+    int64_t offset;
+    uint32_t extra_data_size;
    int ret;

    if (!s->nb_snapshots) {
@@ -99,16 +64,10 @@ static int qcow2_do_read_snapshots(BlockDriverState *bs, bool repair,
    s->snapshots = g_new0(QCowSnapshot, s->nb_snapshots);

    for(i = 0; i < s->nb_snapshots; i++) {
-        bool truncate_unknown_extra_data = false;
-
-        pre_sn_offset = offset;
-        table_length = ROUND_UP(table_length, 8);
-
        /* Read statically sized part of the snapshot header */
        offset = ROUND_UP(offset, 8);
        ret = bdrv_pread(bs->file, offset, &h, sizeof(h));
        if (ret < 0) {
-            error_setg_errno(errp, -ret, "Failed to read snapshot table");
            goto fail;
        }

@@ -120,77 +79,33 @@ static int qcow2_do_read_snapshots(BlockDriverState *bs, bool repair,
        sn->date_sec = be32_to_cpu(h.date_sec);
        sn->date_nsec = be32_to_cpu(h.date_nsec);
        sn->vm_clock_nsec = be64_to_cpu(h.vm_clock_nsec);
-        sn->extra_data_size = be32_to_cpu(h.extra_data_size);
+        extra_data_size = be32_to_cpu(h.extra_data_size);

        id_str_size = be16_to_cpu(h.id_str_size);
        name_size = be16_to_cpu(h.name_size);

-        if (sn->extra_data_size > QCOW_MAX_SNAPSHOT_EXTRA_DATA) {
-            if (!repair) {
-                ret = -EFBIG;
-                error_setg(errp, "Too much extra metadata in snapshot table "
-                           "entry %i", i);
-                error_append_hint(errp, "You can force-remove this extra "
-                                  "metadata with qemu-img check -r all\n");
-                goto fail;
-            }
-
-            fprintf(stderr, "Discarding too much extra metadata in snapshot "
-                    "table entry %i (%" PRIu32 " > %u)\n",
-                    i, sn->extra_data_size, QCOW_MAX_SNAPSHOT_EXTRA_DATA);
-
-            (*extra_data_dropped)++;
-            truncate_unknown_extra_data = true;
-        }
-
-        /* Read known extra data */
+        /* Read extra data */
        ret = bdrv_pread(bs->file, offset, &extra,
-                         MIN(sizeof(extra), sn->extra_data_size));
+                         MIN(sizeof(extra), extra_data_size));
        if (ret < 0) {
-            error_setg_errno(errp, -ret, "Failed to read snapshot table");
            goto fail;
        }
-        offset += MIN(sizeof(extra), sn->extra_data_size);
+        offset += extra_data_size;

-        if (sn->extra_data_size >= endof(QCowSnapshotExtraData,
-                                         vm_state_size_large)) {
+        if (extra_data_size >= 8) {
            sn->vm_state_size = be64_to_cpu(extra.vm_state_size_large);
        }

-        if (sn->extra_data_size >= endof(QCowSnapshotExtraData, disk_size)) {
+        if (extra_data_size >= 16) {
            sn->disk_size = be64_to_cpu(extra.disk_size);
        } else {
            sn->disk_size = bs->total_sectors * BDRV_SECTOR_SIZE;
        }

-        if (sn->extra_data_size > sizeof(extra)) {
-            uint64_t extra_data_end;
-            size_t unknown_extra_data_size;
-
-            extra_data_end = offset + sn->extra_data_size - sizeof(extra);
-
-            if (truncate_unknown_extra_data) {
-                sn->extra_data_size = QCOW_MAX_SNAPSHOT_EXTRA_DATA;
-            }
-
-            /* Store unknown extra data */
-            unknown_extra_data_size = sn->extra_data_size - sizeof(extra);
-            sn->unknown_extra_data = g_malloc(unknown_extra_data_size);
-            ret = bdrv_pread(bs->file, offset, sn->unknown_extra_data,
-                             unknown_extra_data_size);
-            if (ret < 0) {
-                error_setg_errno(errp, -ret,
-                                 "Failed to read snapshot table");
-                goto fail;
-            }
-            offset = extra_data_end;
-        }
-
        /* Read snapshot ID */
        sn->id_str = g_malloc(id_str_size + 1);
        ret = bdrv_pread(bs->file, offset, sn->id_str, id_str_size);
        if (ret < 0) {
-            error_setg_errno(errp, -ret, "Failed to read snapshot table");
            goto fail;
        }
        offset += id_str_size;
@@ -200,47 +115,14 @@ static int qcow2_do_read_snapshots(BlockDriverState *bs, bool repair,
        sn->name = g_malloc(name_size + 1);
        ret = bdrv_pread(bs->file, offset, sn->name, name_size);
        if (ret < 0) {
-            error_setg_errno(errp, -ret, "Failed to read snapshot table");
            goto fail;
        }
        offset += name_size;
        sn->name[name_size] = '\0';

-        /* Note that the extra data may have been truncated */
-        table_length += sizeof(h) + sn->extra_data_size + id_str_size +
-                        name_size;
-        if (!repair) {
-            assert(table_length == offset - s->snapshots_offset);
-        }
-
-        if (table_length > QCOW_MAX_SNAPSHOTS_SIZE ||
-            offset - s->snapshots_offset > INT_MAX)
-        {
-            if (!repair) {
-                ret = -EFBIG;
-                error_setg(errp, "Snapshot table is too big");
-                error_append_hint(errp, "You can force-remove all %u "
-                                  "overhanging snapshots with qemu-img check "
-                                  "-r all\n", s->nb_snapshots - i);
-                goto fail;
-            }
-
-            fprintf(stderr, "Discarding %u overhanging snapshots (snapshot "
-                    "table is too big)\n", s->nb_snapshots - i);
-
-            *nb_clusters_reduced += (s->nb_snapshots - i);
-
-            /* Discard current snapshot also */
-            qcow2_free_single_snapshot(bs, i);
-
-            /*
-             * This leaks all the rest of the snapshot table and the
-             * snapshots' clusters, but we run in check -r all mode,
-             * so qcow2_check_refcounts() will take care of it.
-             */
-            s->nb_snapshots = i;
-            offset = pre_sn_offset;
-            break;
+        if (offset - s->snapshots_offset > QCOW_MAX_SNAPSHOTS_SIZE) {
+            ret = -EFBIG;
+            goto fail;
        }
    }

@@ -253,13 +135,8 @@ fail:
    return ret;
 }

-int qcow2_read_snapshots(BlockDriverState *bs, Error **errp)
-{
-    return qcow2_do_read_snapshots(bs, false, NULL, NULL, errp);
-}
-
 /* add at the end of the file a new list of snapshots */
-int qcow2_write_snapshots(BlockDriverState *bs)
+static int qcow2_write_snapshots(BlockDriverState *bs)
 {
    BDRVQcow2State *s = bs->opaque;
    QCowSnapshot *sn;
@@ -279,7 +156,7 @@ int qcow2_write_snapshots(BlockDriverState *bs)
        sn = s->snapshots + i;
        offset = ROUND_UP(offset, 8);
        offset += sizeof(h);
-        offset += MAX(sizeof(extra), sn->extra_data_size);
+        offset += sizeof(extra);
        offset += strlen(sn->id_str);
        offset += strlen(sn->name);

@@ -326,8 +203,7 @@ int qcow2_write_snapshots(BlockDriverState *bs)
        h.date_sec = cpu_to_be32(sn->date_sec);
        h.date_nsec = cpu_to_be32(sn->date_nsec);
        h.vm_clock_nsec = cpu_to_be64(sn->vm_clock_nsec);
-        h.extra_data_size = cpu_to_be32(MAX(sizeof(extra),
-                                            sn->extra_data_size));
+        h.extra_data_size = cpu_to_be32(sizeof(extra));

        memset(&extra, 0, sizeof(extra));
        extra.vm_state_size_large = cpu_to_be64(sn->vm_state_size);
@@ -352,22 +228,6 @@ int qcow2_write_snapshots(BlockDriverState *bs)
        }
        offset += sizeof(extra);

-        if (sn->extra_data_size > sizeof(extra)) {
-            size_t unknown_extra_data_size =
-                sn->extra_data_size - sizeof(extra);
-
-            /* qcow2_read_snapshots() ensures no unbounded allocation */
-            assert(unknown_extra_data_size <= BDRV_REQUEST_MAX_BYTES);
-            assert(sn->unknown_extra_data);
-
-            ret = bdrv_pwrite(bs->file, offset, sn->unknown_extra_data,
-                              unknown_extra_data_size);
-            if (ret < 0) {
-                goto fail;
-            }
-            offset += unknown_extra_data_size;
-        }
-
        ret = bdrv_pwrite(bs->file, offset, sn->id_str, id_str_size);
        if (ret < 0) {
            goto fail;
@@ -391,7 +251,7 @@ int qcow2_write_snapshots(BlockDriverState *bs)
    }

    QEMU_BUILD_BUG_ON(offsetof(QCowHeader, snapshots_offset) !=
-                      endof(QCowHeader, nb_snapshots));
+        offsetof(QCowHeader, nb_snapshots) + sizeof(header_data.nb_snapshots));

    header_data.nb_snapshots        = cpu_to_be32(s->nb_snapshots);
    header_data.snapshots_offset    = cpu_to_be64(snapshots_offset);
@@ -417,151 +277,6 @@ fail:
    return ret;
 }

-int coroutine_fn qcow2_check_read_snapshot_table(BlockDriverState *bs,
-                                                 BdrvCheckResult *result,
-                                                 BdrvCheckMode fix)
-{
-    BDRVQcow2State *s = bs->opaque;
-    Error *local_err = NULL;
-    int nb_clusters_reduced = 0;
-    int extra_data_dropped = 0;
-    int ret;
-    struct {
-        uint32_t nb_snapshots;
-        uint64_t snapshots_offset;
-    } QEMU_PACKED snapshot_table_pointer;
-
-    /* qcow2_do_open() discards this information in check mode */
-    ret = bdrv_pread(bs->file, offsetof(QCowHeader, nb_snapshots),
-                     &snapshot_table_pointer, sizeof(snapshot_table_pointer));
-    if (ret < 0) {
-        result->check_errors++;
-        fprintf(stderr, "ERROR failed to read the snapshot table pointer from "
-                "the image header: %s\n", strerror(-ret));
-        return ret;
-    }
-
-    s->snapshots_offset = be64_to_cpu(snapshot_table_pointer.snapshots_offset);
-    s->nb_snapshots = be32_to_cpu(snapshot_table_pointer.nb_snapshots);
-
-    if (s->nb_snapshots > QCOW_MAX_SNAPSHOTS && (fix & BDRV_FIX_ERRORS)) {
-        fprintf(stderr, "Discarding %u overhanging snapshots\n",
-                s->nb_snapshots - QCOW_MAX_SNAPSHOTS);
-
-        nb_clusters_reduced += s->nb_snapshots - QCOW_MAX_SNAPSHOTS;
-        s->nb_snapshots = QCOW_MAX_SNAPSHOTS;
-    }
-
-    ret = qcow2_validate_table(bs, s->snapshots_offset, s->nb_snapshots,
-                               sizeof(QCowSnapshotHeader),
-                               sizeof(QCowSnapshotHeader) * QCOW_MAX_SNAPSHOTS,
-                               "snapshot table", &local_err);
-    if (ret < 0) {
-        result->check_errors++;
-        error_reportf_err(local_err, "ERROR ");
-
-        if (s->nb_snapshots > QCOW_MAX_SNAPSHOTS) {
-            fprintf(stderr, "You can force-remove all %u overhanging snapshots "
-                    "with qemu-img check -r all\n",
-                    s->nb_snapshots - QCOW_MAX_SNAPSHOTS);
-        }
-
-        /* We did not read the snapshot table, so invalidate this information */
-        s->snapshots_offset = 0;
-        s->nb_snapshots = 0;
-
-        return ret;
-    }
-
-    qemu_co_mutex_unlock(&s->lock);
-    ret = qcow2_do_read_snapshots(bs, fix & BDRV_FIX_ERRORS,
-                                  &nb_clusters_reduced, &extra_data_dropped,
-                                  &local_err);
-    qemu_co_mutex_lock(&s->lock);
-    if (ret < 0) {
-        result->check_errors++;
-        error_reportf_err(local_err,
-                          "ERROR failed to read the snapshot table: ");
-
-        /* We did not read the snapshot table, so invalidate this information */
-        s->snapshots_offset = 0;
-        s->nb_snapshots = 0;
-
-        return ret;
-    }
-    result->corruptions += nb_clusters_reduced + extra_data_dropped;
-
-    if (nb_clusters_reduced) {
-        /*
-         * Update image header now, because:
-         * (1) qcow2_check_refcounts() relies on s->nb_snapshots to be
-         *     the same as what the image header says,
-         * (2) this leaks clusters, but qcow2_check_refcounts() will
-         *     fix that.
-         */
-        assert(fix & BDRV_FIX_ERRORS);
-
-        snapshot_table_pointer.nb_snapshots = cpu_to_be32(s->nb_snapshots);
-        ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, nb_snapshots),
-                               &snapshot_table_pointer.nb_snapshots,
-                               sizeof(snapshot_table_pointer.nb_snapshots));
-        if (ret < 0) {
-            result->check_errors++;
-            fprintf(stderr, "ERROR failed to update the snapshot count in the "
-                    "image header: %s\n", strerror(-ret));
-            return ret;
-        }
-
-        result->corruptions_fixed += nb_clusters_reduced;
-        result->corruptions -= nb_clusters_reduced;
-    }
-
-    /*
-     * All of v3 images' snapshot table entries need to have at least
-     * 16 bytes of extra data.
-     */
-    if (s->qcow_version >= 3) {
-        int i;
-        for (i = 0; i < s->nb_snapshots; i++) {
-            if (s->snapshots[i].extra_data_size <
-                sizeof_field(QCowSnapshotExtraData, vm_state_size_large) +
-                sizeof_field(QCowSnapshotExtraData, disk_size))
-            {
-                result->corruptions++;
-                fprintf(stderr, "%s snapshot table entry %i is incomplete\n",
-                        fix & BDRV_FIX_ERRORS ? "Repairing" : "ERROR", i);
-            }
-        }
-    }
-
-    return 0;
-}
-
-int coroutine_fn qcow2_check_fix_snapshot_table(BlockDriverState *bs,
-                                                BdrvCheckResult *result,
-                                                BdrvCheckMode fix)
-{
-    BDRVQcow2State *s = bs->opaque;
-    int ret;
-
-    if (result->corruptions && (fix & BDRV_FIX_ERRORS)) {
-        qemu_co_mutex_unlock(&s->lock);
-        ret = qcow2_write_snapshots(bs);
-        qemu_co_mutex_lock(&s->lock);
-        if (ret < 0) {
-            result->check_errors++;
-            fprintf(stderr, "ERROR failed to update snapshot table: %s\n",
-                    strerror(-ret));
-            return ret;
-        }
-
-        result->corruptions_fixed += result->corruptions;
-        result->corruptions = 0;
-    }
-
-    return 0;
-}
-
 static void find_new_snapshot_id(BlockDriverState *bs,
                                 char *id_str, int id_str_size)
 {
@@ -655,7 +370,6 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
    sn->date_sec = sn_info->date_sec;
    sn->date_nsec = sn_info->date_nsec;
    sn->vm_clock_nsec = sn_info->vm_clock_nsec;
-    sn->extra_data_size = sizeof(QCowSnapshotExtraData);

    /* Allocate the L1 table of the snapshot and copy the current one there. */
    l1_table_offset = qcow2_alloc_clusters(bs, s->l1_size * sizeof(uint64_t));
@@ -927,7 +641,6 @@ int qcow2_snapshot_delete(BlockDriverState *bs,
     * The snapshot is now unused, clean up. If we fail after this point, we
     * won't recover but just leak clusters.
     */
-    g_free(sn.unknown_extra_data);
    g_free(sn.id_str);
    g_free(sn.name);

@@ -234,70 +234,35 @@ static int qcow2_encdec_pool_func(void *opaque)
 }

 static int coroutine_fn
-qcow2_co_encdec(BlockDriverState *bs, uint64_t host_offset,
-                uint64_t guest_offset, void *buf, size_t len,
-                Qcow2EncDecFunc func)
+qcow2_co_encdec(BlockDriverState *bs, uint64_t file_cluster_offset,
+                  uint64_t offset, void *buf, size_t len, Qcow2EncDecFunc func)
 {
    BDRVQcow2State *s = bs->opaque;
    Qcow2EncDecData arg = {
        .block = s->crypto,
-        .offset = s->crypt_physical_offset ? host_offset : guest_offset,
+        .offset = s->crypt_physical_offset ?
+                      file_cluster_offset + offset_into_cluster(s, offset) :
+                      offset,
        .buf = buf,
        .len = len,
        .func = func,
    };

-    assert(QEMU_IS_ALIGNED(guest_offset, BDRV_SECTOR_SIZE));
-    assert(QEMU_IS_ALIGNED(host_offset, BDRV_SECTOR_SIZE));
-    assert(QEMU_IS_ALIGNED(len, BDRV_SECTOR_SIZE));
-    assert(s->crypto);
-
-    return len == 0 ? 0 : qcow2_co_process(bs, qcow2_encdec_pool_func, &arg);
+    return qcow2_co_process(bs, qcow2_encdec_pool_func, &arg);
 }

-/*
- * qcow2_co_encrypt()
- *
- * Encrypts one or more contiguous aligned sectors
- *
- * @host_offset - underlying storage offset of the first sector of the
- * data to be encrypted
- *
- * @guest_offset - guest (virtual) offset of the first sector of the
- * data to be encrypted
- *
- * @buf - buffer with the data to encrypt, that after encryption
- *        will be written to the underlying storage device at
- *        @host_offset
- *
- * @len - length of the buffer (must be a BDRV_SECTOR_SIZE multiple)
- *
- * Depending on the encryption method, @host_offset and/or @guest_offset
- * may be used for generating the initialization vector for
- * encryption.
- *
- * Note that while the whole range must be aligned on sectors, it
- * does not have to be aligned on clusters and can also cross cluster
- * boundaries
- */
 int coroutine_fn
-qcow2_co_encrypt(BlockDriverState *bs, uint64_t host_offset,
-                 uint64_t guest_offset, void *buf, size_t len)
+qcow2_co_encrypt(BlockDriverState *bs, uint64_t file_cluster_offset,
+                 uint64_t offset, void *buf, size_t len)
 {
-    return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,
-                           qcrypto_block_encrypt);
+    return qcow2_co_encdec(bs, file_cluster_offset, offset, buf, len,
+                             qcrypto_block_encrypt);
 }

-/*
- * qcow2_co_decrypt()
- *
- * Decrypts one or more contiguous aligned sectors
- * Similar to qcow2_co_encrypt
- */
 int coroutine_fn
-qcow2_co_decrypt(BlockDriverState *bs, uint64_t host_offset,
-                 uint64_t guest_offset, void *buf, size_t len)
+qcow2_co_decrypt(BlockDriverState *bs, uint64_t file_cluster_offset,
+                 uint64_t offset, void *buf, size_t len)
 {
-    return qcow2_co_encdec(bs, host_offset, guest_offset, buf, len,
-                           qcrypto_block_decrypt);
+    return qcow2_co_encdec(bs, file_cluster_offset, offset, buf, len,
+                             qcrypto_block_decrypt);
 }
@@ -1 +1 @@
 .1.90
 .1.1