Re: Using rsync to maintain local FC10 updates repo

Robert Moskowitz wrote:

So I wanted to do a simple rsync against one of the mirrors, and tried the script:

#!/bin/sh

rsync -auv rsync://mirrors.usc.edu/pub/linux/distributions/fedora/linux/updates/10/i386/ \

--delete --exclude=debug/ /repos/fedora/10/updates/i386

The rsync paths aren't like the ftp or http paths.  You want:

#!/bin/sh
rsync -auv rsync://mirrors.usc.edu/fedora/linux/updates/10/i386/ \
--delete --exclude=debug/ /repos/fedora/10/updates/i386

Except that this mirror's rsync seems to be broken!  ;)

#!/bin/sh


MIRROR=/home/gordon/deploy/mirrors
SYNC="rsync -avH --delete --exclude debug --exclude '*-debuginfo-*.rpm'"

fail_cd () {
    echo "Could not cd to $1"
    exit 1
}

usage () {
	cat <<EOF
Use: update_mirror [config_file] [command]
Command must be one of "init", or "update".
EOF
	exit
}

init () {
	# Init will reinitialize all of the mirrors.
	for ARCH in $ARCHLIST ; do
		echo "Initializing ${NAME} release ${RELEASE} for ${ARCH}"
		BASEPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${BASE}/"
		echo "Downloading base OS to ${BASEPATH}"
		mkdir -p ${MIRROR}/${NAME}-${RELEASE}-${ARCH}
		eval URL="\$URL_${BASE}"
		eval $SYNC ${URL} ${BASEPATH}
		# Build a list of packages in the base for update-only repos
		BASEPKGSYNCLIST=${BASEPATH%/}.synclist
		find ${BASEPATH} -type f -a -name '*.rpm' \
			| xargs rpm -qp --qf "+ %{NAME}-*.rpm\n" \
			> ${BASEPKGSYNCLIST}
		for REPO in $ADDITIONAL ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 0
		done
		for REPO in $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 1
		done
	done
}

sync_additional () {
	UPDATEONLY="$1"
	if [ "$UPDATEONLY" = "1" -a -r "${BASEPKGSYNCLIST}" ]; then
		SYNCLIST="--include-from ${BASEPKGSYNCLIST} --include '*/' --include 'repodata/**' --exclude '*'"
	fi
	echo "Updating \"${REPO}\" in ${REPOPATH}"
	eval URL="\$URL_${REPO}"
	eval $SYNC ${SYNCLIST} ${URL} ${REPOPATH}
	if test $? -ne 0 ; then
		echo "Update for \"${REPO}\" failed, aborting script"
		exit 0
	fi
	unset SYNCLIST
	# Set the marker if anything was actually downloaded.  This marker
	# and the beacon file were created in the update function.
	UPDATECOUNT=$(find "${REPOPATH}" -type f -cnewer "${BEACONPATH}" | wc -l)
	test $UPDATECOUNT -gt 0 && UPDATESFOUND=1
}

update () {
	for ARCH in $ARCHLIST ; do
		echo "Updating ${NAME} release ${RELEASE} for ${ARCH}"
		BASEPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${BASE}/"
		BASEPKGSYNCLIST=${BASEPATH%/}.synclist
		RESULTPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/result/"
		BEACONPATH="$(mktemp /tmp/beacon.XXXXXX)"
		UPDATESFOUND=0
		for REPO in $ADDITIONAL ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 0
		done
		for REPO in $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 1
		done
		rm "${BEACONPATH}"
		if test "${UPDATESFOUND}" -eq 0 ; then
			echo "No new packages were found.  Skipping repomd update."
			continue
		fi
		echo "Cleaning up result directory"
		test -e ${RESULTPATH}/RPMS && rm -rf ${RESULTPATH}/RPMS
		mkdir -p ${RESULTPATH}/{repodata,images/xen,RPMS}
		ln -f ${BASEPATH}/repodata/*comps.xml ${RESULTPATH}/repodata/
		ln -f ${BASEPATH}/images/*.img ${RESULTPATH}/images/
		ln -f ${BASEPATH}/images/xen/* ${RESULTPATH}/images/xen/
		for REPO in $ADDITIONAL $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			echo "Removing packages which exist in \"${REPO}\" from base"
			# If any package in this repository exists in the base, remove it.
			BASEPKGLIST=$(mktemp /tmp/baselist.XXXXXX)
			REPOPKGLIST=$(mktemp /tmp/repolist.XXXXXX)
			find ${BASEPATH} -type f -a -name '*.rpm' \
				| xargs rpm -qp --qf "%{NAME}:%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm\n" \
				> ${BASEPKGLIST}
			find ${REPOPATH} -type f -a -name '*.rpm' \
				| xargs rpm -qp --qf "%{NAME}\n" \
				| sort | uniq \
				> ${REPOPKGLIST}
			egrep "^("$(tr '\n' '|' < ${REPOPKGLIST})"):" ${BASEPKGLIST} \
				| cut -f2- -d: \
				| while read file ; do
					find ${BASEPATH} -type f -a -name "${file}"
				done \
				| xargs --no-run-if-empty rm
			rm -f ${BASEPKGLIST} ${REPOPKGLIST}
			find ${REPOPATH} -type f -a -name '*.rpm' \
				| xargs --no-run-if-empty ln -t ${RESULTPATH}/RPMS/
		done
		find ${BASEPATH} -type f -a -name '*.rpm' \
			| xargs --no-run-if-empty ln -t ${RESULTPATH}/RPMS/
		echo "Updating repomd data in ${RESULTPATH}"
		# Update the repodata that anaconda uses.
		pushd ${RESULTPATH}
			createrepo -g repodata/*comps.xml ${RESULTPATH}
		popd
	done
}

test -f "$1" || usage
. $1

case $2 in
	init)
		init
		;;
	update)
		update
		;;
	*)
		usage
		;;
esac

exit

NAME=fedora
RELEASE=10
ARCHLIST="i386 x86_64"
ARCHLIST="i386"
BASE="os"
UPDATES="updates"
URL_os='rsync://mirrors.kernel.org/fedora/releases/test/10-Preview/Fedora/${ARCH}/os/'
URL_updates='rsync://mirrors.kernel.org/fedora/development/${ARCH}/os/'
#URL_os='rsync://mirrors.kernel.org/fedora/releases/${RELEASE}/Fedora/${ARCH}/os/'
#URL_updates='rsync://mirrors.kernel.org/fedora/updates/${RELEASE}/${ARCH}/'
#URL_os='rsync://mirror.anl.gov/fedora/linux/releases/${RELEASE}/Fedora/${ARCH}/os/'
#URL_updates='rsync://mirror.anl.gov/fedora/linux/updates/${RELEASE}/${ARCH}/'

-- 
fedora-list mailing list
fedora-list@xxxxxxxxxx
To unsubscribe: https://www.redhat.com/mailman/listinfo/fedora-list
Guidelines: http://fedoraproject.org/wiki/Communicate/MailingListGuidelines