dak/docs/import-into-debian-archive.txt

ROOT=/srv/archive.debian.org
etcdir=${ROOT:?}/etc
logdir=${ROOT:?}/log

export PATH="${ROOT}/bin:$PATH"
#export RSYNC_SSL_PORT=1873
#export RSYNC_SSL_CA_CERT=/etc/ssl/ca-debian/ca-certificates.crt
#export RSYNC_USER=...
#export RSYNC_PASSWORD=...

cd ${logdir}
dry_run=1
time debmirror \
  ${dry_run:+--dry-run} \
  --progress \
  --nocleanup \
  --method=rsync --user="${RSYNC_USER:?}" --host=security-master.debian.org --root=debian-security \
  --omit-suite-symlinks \
  --dist=buster/updates \
  --arch=amd64,arm64,armhf,i386 \
  --section=main,contrib,non-free,main/debian-installer,contrib/debian-installer,non-free/debian-installer \
  --i18n --getcontents \
  --checksums \
  --check-gpg --keyring=/usr/share/keyrings/debian-archive-buster-security-automatic.gpg \
  --diff=none --rsync-extra=none \
  --rsync-batch=1000 \
  ${ROOT:?}/debian-archive/debian-security \
  > ${logdir:?}/2025-06-25-debmirror-buster-security${dry_run:+-dry-run}.txt 2>&1

(
  # sync dists/ stuff again to make sure by-hash/ gets imported too.
  set -e
  cd ${ROOT:?}/debian-archive/debian-security/dists/buster/updates
  rm -rf main contrib non-free non-free-firmware
  rsync -aL ${RSYNC_USER}@security-master.debian.org::debian-security/dists/buster/updates/{main,contrib,non-free,non-free-firmware} .
)

# For later:
cd ${logdir}
suite=bullseye keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=(--di-dist=dists --di-arch=arches)
suite=bullseye-proposed-updates keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
suite=bullseye-updates keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
suite=bullseye-backports keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
suite=bullseye-backports-sloppy keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
dry_run=1
time debmirror \
  ${dry_run:+--dry-run} \
  --progress \
  --nocleanup \
  --method=rsync ${RSYNC_USER:+--user="${RSYNC_USER:?}"} --host=127.0.0.1 --root=debian \
  --omit-suite-symlinks \
  --dist=${suite:?} \
  --arch=amd64,arm64,armel,armhf,i386,mips64el,mipsel,ppc64el,s390x \
  "${di_opts[@]}" \
  --section=main,contrib,non-free,main/debian-installer,contrib/debian-installer,non-free/debian-installer \
  --i18n --getcontents \
  --checksums \
  --check-gpg --keyring=${keyring:?} \
  --diff=none --rsync-extra=none \
  --rsync-batch=1000 \
  ${ROOT:?}/debian-archive/debian \
  > ${logdir:?}/2025-06-16-debmirror-${suite:?}${dry_run:+-dry-run}.txt 2>&1

# Copied dists/ tree as otherwise older installer versions and ChangeLog were missing.
# Only difference after this: Release{,.gpg} are real files, not symlinks to zzz-dists
for suite in bullseye{,-proposed-updates,-updates,-backports,-backports-sloppy}; do
  (
  set -e
  cd ${ROOT:?}/debian-archive/debian/dists/${suite:?}
  rm -rf main contrib non-free
  cp -a /srv/ftp-master.debian.org/ftp/dists/${suite:?}/{main,contrib,non-free} .
  if [ "$suite" = bullseye ]; then
    cp -a /srv/ftp-master.debian.org/ftp/dists/${suite:?}/ChangeLog .
  fi
  if [ "$suite" = bullseye-proposed-updates ]; then
    cp -a /srv/ftp-master.debian.org/ftp/dists/${suite:?}/*.changes .
  fi
  )
done

# jdupes if not importing other stuff; see below

# check DEP11 data?
# + stretch-debug

cd ${logdir}

suite=bullseye-debug keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
suite=bullseye-proposed-updates-debug keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
suite=bullseye-backports-debug keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
suite=bullseye-backports-sloppy-debug keyring=/usr/share/keyrings/debian-archive-bullseye-automatic.gpg di_opts=()
dry_run=1
time debmirror \
  ${dry_run:+--dry-run} \
  --progress \
  --nocleanup \
  --method=rsync ${RSYNC_USER:+--user="${RSYNC_USER:?}"} --host=127.0.0.1 --root=debian-debug \
  --omit-suite-symlinks \
  --dist=${suite:?} \
  --arch=amd64,arm64,armel,armhf,i386,mips64el,mipsel,ppc64el,s390x \
  "${di_opts[@]}" \
  --section=main,contrib,non-free,main/debian-installer,contrib/debian-installer,non-free/debian-installer \
  --i18n --getcontents \
  --checksums \
  --check-gpg --keyring=${keyring:?} \
  --diff=none --rsync-extra=none \
  --rsync-batch=1000 \
  ${ROOT:?}/debian-archive/debian-debug \
  > ${logdir:?}/2025-06-16-debmirror-${suite:?}${dry_run:+-dry-run}.txt 2>&1

cd ${ROOT:?}/debian-archive/debian/dists
for d in bullseye{,-updates,-proposed-updates,-backports,-backports-sloppy}; do
  (
  set -e
  cd ${d}
  rm -rf main contrib non-free non-free-firmware
  /usr/bin/rsync -a /srv/ftp-master.debian.org/ftp/dists/${d}/{main,contrib,non-free} .
  if [ -e /srv/ftp-master.debian.org/ftp/dists/${d}/non-free-firmware ]; then
    /usr/bin/rsync -a /srv/ftp-master.debian.org/ftp/dists/${d}/non-free-firmware .
  fi
  )
done

cd ${ROOT:?}/debian-archive/debian-debug/dists
for d in bullseye{,-proposed-updates,-backports,-backports-sloppy}-debug; do
  (
  set -e
  cd ${d}
  rm -rf main contrib non-free non-free-firmware
  /usr/bin/rsync -a /srv/ftp-master.debian.org/archive/debian-debug/dists/${d}/{main,contrib,non-free} .
  if [ -e /srv/ftp-master.debian.org/archive/debian-debug/dists/${d}/non-free-firmware ]; then
    /usr/bin/rsync -a /srv/ftp-master.debian.org/archive/debian-debug/dists/${d}/non-free-firmware .
  fi
  )
done

# check that for each architecture at least one binary ended up in pool/.
# debmirror 1:2.35+deb11u1 doesn't seem to complain if we specified the wrong architectures.

# jdupes: --order=time to get timestamp of oldest file
# Note: jdupes feels veeeery slow (hours): it reads beginning of all files
# first, but then identifying potential copies is fast. There is no feedback
# while it reads data.
cd ${ROOT}
time jdupes --order=time --printwithsummary --recurse debian-archive > ${logdir}/2025-06-25-archive-jdupes-test.txt 2>&1
time jdupes --order=time --linkhard --recurse debian-archive > ${logdir}/2025-06-25-archive-jdupes.txt 2>&1

date > ${ROOT:?}/debian-archive/project/trace/master

# as dak:
sudo -u dak sudo -u archvsync /home/archvsync/bin/runmirrors archive

# Remind webmaster to update the corresponding pages in www.debian.org and remove stuff from packages.d.o