Refactor fetch and repack
This commit is contained in:
root
parent
a50a225ff0
commit
84ac13f5a9
1 changed files with 100 additions and 131 deletions
|
|
@ -88,34 +88,21 @@ splitcolon()
|
||||||
setvar "$3" "${1#*:}"
|
setvar "$3" "${1#*:}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# if $1 contains $2
|
|
||||||
contains()
|
|
||||||
{
|
|
||||||
isnull "${1##*"$2"*}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Pick words from each line
|
# Pick words from each line
|
||||||
# $1 return variable name
|
# $1 return variable name
|
||||||
# $2 field list "1,2,3"
|
# $2 input value
|
||||||
# $3 input value
|
pick_fields_1_2()
|
||||||
pick_fields()
|
|
||||||
{
|
{
|
||||||
local f_line_= f_result_= f_mask_= f_ret_var= f_oifs="$IFS" IFS=
|
local f_ret= f_line= f_var= f_oifs="$IFS" IFS=
|
||||||
f_ret_var=$1
|
f_var=$1
|
||||||
f_mask_=$2
|
|
||||||
IFS=$Newline
|
IFS=$Newline
|
||||||
for f_line_ in $3
|
for f_line in $2
|
||||||
do
|
do
|
||||||
IFS=$f_oifs
|
IFS=$f_oifs
|
||||||
# split $f_line_ into words and pick them out
|
set -- $f_line
|
||||||
set -- $f_line_
|
f_ret=$f_ret"${1:-} ${2:-}"$Newline
|
||||||
f_line_=
|
|
||||||
! contains "$f_mask_" 1 || f_line_=${1:-}
|
|
||||||
! contains "$f_mask_" 2 || f_line_="$f_line_ ${2:-}"
|
|
||||||
! contains "$f_mask_" 3 || f_line_="$f_line_ ${3:-}"
|
|
||||||
append_to @f_result_ "${f_line_# }"
|
|
||||||
done
|
done
|
||||||
setvar "$f_ret_var" "$f_result_"
|
setvar "$f_var" "${f_ret#$Newline}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Take all lines matching $2 (full line)
|
# Take all lines matching $2 (full line)
|
||||||
|
|
@ -557,37 +544,60 @@ ensure_connected()
|
||||||
isnull "$r_name" || git config "remote.$r_name.gcrypt-id" "$r_repoid"
|
isnull "$r_name" || git config "remote.$r_name.gcrypt-id" "$r_repoid"
|
||||||
}
|
}
|
||||||
|
|
||||||
# $1 is the packline pack :SHA256:abc1231..
|
# $1 is the hash type (SHA256 etc)
|
||||||
fetch_decrypt_pack()
|
# $2 the pack id
|
||||||
|
# $3 the key
|
||||||
|
get_verify_decrypt_pack()
|
||||||
{
|
{
|
||||||
local rcv_id= r_key= r_htype= r_pack=
|
local rcv_id= tmp_encrypted=
|
||||||
splitcolon "${1#pack :}" @r_htype @r_pack
|
tmp_encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$"
|
||||||
|
GET "$URL" "$2" "$tmp_encrypted" &&
|
||||||
if isnoteq "$r_htype" SHA256 && isnoteq "$r_htype" SHA224 &&
|
rcv_id=$(gpg_hash "$1" < "$tmp_encrypted") &&
|
||||||
isnoteq "$r_htype" SHA384 && isnoteq "$r_htype" SHA512
|
iseq "$rcv_id" "$2" || echo_die "Packfile $2 does not match digest!"
|
||||||
then
|
DECRYPT "$3" < "$tmp_encrypted"
|
||||||
echo_die "Packline malformed: $1"
|
rm -f "$tmp_encrypted"
|
||||||
fi
|
|
||||||
GET "$URL" "$r_pack" "$TmpPack_Encrypted" &&
|
|
||||||
rcv_id=$(gpg_hash "$r_htype" < "$TmpPack_Encrypted") &&
|
|
||||||
iseq "$rcv_id" "$r_pack" ||
|
|
||||||
echo_die "Packfile $r_pack does not match digest!"
|
|
||||||
filter_to @r_key "pack :${r_htype}:$r_pack *" "$Packlist"
|
|
||||||
pick_fields @r_key 3 "$r_key"
|
|
||||||
DECRYPT "$r_key" < "$TmpPack_Encrypted"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# $1 is new pack id $2 key, $3, $4 return variables
|
# download all packlines (pack :SHA256:a32abc1231) from stdin (or die)
|
||||||
# set $3 to yes if repacked
|
# $1 destdir (when repack, else "")
|
||||||
# $4 to list of packfiles to delete
|
get_pack_files()
|
||||||
|
{
|
||||||
|
local pack_id= r_pack_key_line= r_htype= r_pack= key_=
|
||||||
|
while read -r _ pack_id # <<here-document
|
||||||
|
do
|
||||||
|
isnonnull "$pack_id" || continue
|
||||||
|
|
||||||
|
# Get the Packlist line with the key
|
||||||
|
filter_to @r_pack_key_line "pack $pack_id *" "$Packlist"
|
||||||
|
key_=${r_pack_key_line#pack "$pack_id" }
|
||||||
|
|
||||||
|
splitcolon "${pack_id#:}" @r_htype @r_pack
|
||||||
|
if isnonnull "${r_pack##$Hex40*}" ||
|
||||||
|
isnoteq "$r_htype" SHA256 && isnoteq "$r_htype" SHA224 &&
|
||||||
|
isnoteq "$r_htype" SHA384 && isnoteq "$r_htype" SHA512
|
||||||
|
then
|
||||||
|
echo_die "Packline malformed: $pack_id"
|
||||||
|
fi
|
||||||
|
|
||||||
|
get_verify_decrypt_pack "$r_htype" "$r_pack" "$key_" | \
|
||||||
|
if isnull "${1:-}"
|
||||||
|
then
|
||||||
|
# add to local pack list
|
||||||
|
git index-pack -v --stdin >/dev/null
|
||||||
|
xecho "pack $pack_id" >> "$Localdir/have_packs$GITCEPTION"
|
||||||
|
else
|
||||||
|
git index-pack -v --stdin "$1/${r_pack}.pack" >/dev/null
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Download and unpack remote packfiles
|
||||||
|
# $1 is objects tmpfile
|
||||||
|
# $2 return var for list of packfiles to delete
|
||||||
repack_if_needed()
|
repack_if_needed()
|
||||||
{
|
{
|
||||||
local pack_= packline_= premote_= key_= pkeep_= n_= m_= \
|
local n_= m_= kline_= r_line= r_keep_packlist= r_del_list=
|
||||||
orig_ifs= kline_= r_line= r_list_new=
|
|
||||||
|
|
||||||
# $TmpPack_Encrypted set in caller
|
|
||||||
|
|
||||||
setvar "$3" no
|
|
||||||
isnonnull "$Packlist" || return 0
|
isnonnull "$Packlist" || return 0
|
||||||
|
|
||||||
if isnonnull "${GCRYPT_FULL_REPACK:-}"
|
if isnonnull "${GCRYPT_FULL_REPACK:-}"
|
||||||
|
|
@ -596,63 +606,38 @@ repack_if_needed()
|
||||||
Repack_limit=1
|
Repack_limit=1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
pick_fields @premote_ 1,2 "$Packlist"
|
pick_fields_1_2 @r_del_list "$Packlist"
|
||||||
pick_fields @pkeep_ 2 "$Keeplist"
|
|
||||||
|
|
||||||
n_=$(line_count "$Packlist")
|
n_=$(line_count "$Packlist")
|
||||||
m_=$(line_count "$pkeep_")
|
m_=$(line_count "$Keeplist")
|
||||||
if [ "$Repack_limit" -gt "$(($n_ - $m_))" ]; then
|
if iseq 0 "$(( $Repack_limit < ($n_ - $m_) ))"; then
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
echo_info "Repacking remote $NAME, ..."
|
echo_info "Repacking remote $NAME, ..."
|
||||||
|
|
||||||
rm -r -f "$Localdir/pack"
|
rm -r -f "$Localdir/pack"
|
||||||
mkdir -p "$Localdir/pack"
|
mkdir -p "$Localdir/pack"
|
||||||
DECRYPT "$2" < "$TmpPack_Encrypted" |
|
|
||||||
git index-pack -v --stdin "$Localdir/pack/${1}.pack" >/dev/null
|
|
||||||
|
|
||||||
xecho "$premote_" | while read packline_
|
# Split packages to keep and to repack
|
||||||
do
|
if isnonnull "$Keeplist"; then
|
||||||
isnonnull "$packline_" || continue
|
while read -r _ kline_ _ # <<here-document
|
||||||
if isnonnull "$pkeep_" &&
|
|
||||||
xecho "$packline_" | grep -q -e "$pkeep_"
|
|
||||||
then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
pack_=${packline_#$Packpat}
|
|
||||||
fetch_decrypt_pack "$packline_" |
|
|
||||||
git index-pack -v --stdin "$Localdir/pack/${pack_}.pack" >/dev/null
|
|
||||||
done
|
|
||||||
key_=$(genkey "$Packkey_bytes")
|
|
||||||
|
|
||||||
git verify-pack -v "$Localdir"/pack/*.idx | grep -E '^[0-9a-f]{40}' |
|
|
||||||
cut -f 1 -d ' ' |
|
|
||||||
GIT_ALTERNATE_OBJECT_DIRECTORIES=$Localdir \
|
|
||||||
git pack-objects --stdout | ENCRYPT "$key_" > "$TmpPack_Encrypted"
|
|
||||||
|
|
||||||
# Truncate packlist to only the kept packs
|
|
||||||
if isnull "$pkeep_"; then
|
|
||||||
setvar "$4" "$premote_"
|
|
||||||
Packlist=
|
|
||||||
else
|
|
||||||
setvar "$4" "$(xecho "$premote_" | xgrep -v -e "$pkeep_")"
|
|
||||||
orig_ifs=$IFS
|
|
||||||
IFS=$Newline
|
|
||||||
for kline_ in $pkeep_
|
|
||||||
do
|
do
|
||||||
IFS=$orig_ifs
|
isnonnull "$kline_" || continue
|
||||||
filter_to @r_line "pack $kline_ *" "$Packlist"
|
filter_to @r_line "pack $kline_ *" "$Packlist"
|
||||||
append_to @r_list_new "$r_line"
|
append_to @r_keep_packlist "$r_line"
|
||||||
done
|
filter_to ! @r_del_list "pack $kline_" "$r_del_list"
|
||||||
IFS=$orig_ifs
|
done <<EOF
|
||||||
Packlist=$r_list_new
|
$Keeplist
|
||||||
|
EOF
|
||||||
fi
|
fi
|
||||||
|
|
||||||
pack_id=$(pack_hash < "$TmpPack_Encrypted")
|
xecho "$r_del_list" | get_pack_files "$Localdir/pack/"
|
||||||
append_to @Packlist "pack :${Hashtype}:$pack_id $key_"
|
|
||||||
append_to @Keeplist "keep :${Hashtype}:$pack_id 1"
|
git verify-pack -v "$Localdir"/pack/*.idx | grep -E '^[0-9a-f]{40}' |
|
||||||
rm -r -f "$Localdir/pack"
|
cut -f 1 -d ' ' >> "$1"
|
||||||
setvar "$3" yes
|
|
||||||
|
Packlist=$r_keep_packlist
|
||||||
|
setvar "$2" "$r_del_list"
|
||||||
}
|
}
|
||||||
|
|
||||||
do_capabilities()
|
do_capabilities()
|
||||||
|
|
@ -685,23 +670,13 @@ do_list()
|
||||||
|
|
||||||
do_fetch()
|
do_fetch()
|
||||||
{
|
{
|
||||||
# The PACK id is the hash of the encrypted git packfile.
|
# Download packs in the manifest that don't appear in have_packs
|
||||||
# We only download packs mentioned in the encrypted manifest,
|
local pneed_= premote_=
|
||||||
# and check their digest when received.
|
|
||||||
local pack_= packline_= pneed_= premote_=
|
|
||||||
|
|
||||||
ensure_connected
|
ensure_connected
|
||||||
|
|
||||||
if isnull "$Packlist"
|
|
||||||
then
|
|
||||||
echo_git # end with blank line
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
TmpPack_Encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$"
|
|
||||||
|
|
||||||
# The `+` for $GITCEPTION is pointless but we will be safe for stacking
|
# The `+` for $GITCEPTION is pointless but we will be safe for stacking
|
||||||
pick_fields @premote_ 1,2 "$Packlist"
|
pick_fields_1_2 @premote_ "$Packlist"
|
||||||
if [ -s "$Localdir/have_packs+" ]
|
if [ -s "$Localdir/have_packs+" ]
|
||||||
then
|
then
|
||||||
pneed_=$(xecho "$premote_" | xgrep -v -x -f "$Localdir/have_packs+")
|
pneed_=$(xecho "$premote_" | xgrep -v -x -f "$Localdir/have_packs+")
|
||||||
|
|
@ -709,16 +684,8 @@ do_fetch()
|
||||||
pneed_=$premote_
|
pneed_=$premote_
|
||||||
fi
|
fi
|
||||||
|
|
||||||
xecho "$pneed_" | while read packline_
|
xecho "$pneed_" | get_pack_files
|
||||||
do
|
|
||||||
isnonnull "$packline_" || continue
|
|
||||||
fetch_decrypt_pack "$packline_" |
|
|
||||||
git index-pack -v --stdin >/dev/null
|
|
||||||
# add to local pack list
|
|
||||||
xecho "${packline_}" >> "$Localdir/have_packs$GITCEPTION"
|
|
||||||
done
|
|
||||||
|
|
||||||
rm -f "$TmpPack_Encrypted"
|
|
||||||
echo_git # end with blank line
|
echo_git # end with blank line
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -729,8 +696,8 @@ do_push()
|
||||||
# Each git packfile is encrypted and then named for the encrypted
|
# Each git packfile is encrypted and then named for the encrypted
|
||||||
# file's hash. The manifest is updated with the pack id.
|
# file's hash. The manifest is updated with the pack id.
|
||||||
# The manifest is encrypted.
|
# The manifest is encrypted.
|
||||||
local r_revlist= line_= pack_id= key_= obj_= \
|
local r_revlist= line_= pack_id= r_pack_id= key_= obj_= \
|
||||||
r_did_repack= r_pack_delete= r_src= r_dst=
|
r_pack_delete= r_src= r_dst= tmp_encrypted= tmp_objlist=
|
||||||
|
|
||||||
ensure_connected
|
ensure_connected
|
||||||
|
|
||||||
|
|
@ -763,25 +730,26 @@ do_push()
|
||||||
$1
|
$1
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
TmpPack_Encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$"
|
tmp_encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$"
|
||||||
TmpObjlist="$Localdir/tmp_packrevlist.$$"
|
tmp_objlist="$Localdir/tmp_packrevlist.$$"
|
||||||
key_=$(genkey "$Packkey_bytes")
|
|
||||||
|
|
||||||
xecho "$r_revlist" | git rev-list --objects --stdin -- |
|
xecho "$r_revlist" | git rev-list --objects --stdin -- > "$tmp_objlist"
|
||||||
tee "$TmpObjlist" |
|
|
||||||
git pack-objects --stdout | ENCRYPT "$key_">"$TmpPack_Encrypted"
|
|
||||||
# Only send pack if we have any objects to send
|
# Only send pack if we have any objects to send
|
||||||
if [ -s "$TmpObjlist" ]
|
if [ -s "$tmp_objlist" ]
|
||||||
then
|
then
|
||||||
pack_id=$(pack_hash < "$TmpPack_Encrypted")
|
repack_if_needed "$tmp_objlist" @r_pack_delete
|
||||||
repack_if_needed "$pack_id" "$key_" @r_did_repack @r_pack_delete
|
|
||||||
|
|
||||||
if isnoteq "$r_did_repack" yes
|
key_=$(genkey "$Packkey_bytes")
|
||||||
|
GIT_ALTERNATE_OBJECT_DIRECTORIES=$Localdir \
|
||||||
|
git pack-objects --stdout < "$tmp_objlist" |
|
||||||
|
ENCRYPT "$key_" > "$tmp_encrypted"
|
||||||
|
pack_id=$(pack_hash < "$tmp_encrypted")
|
||||||
|
|
||||||
|
append_to @Packlist "pack :${Hashtype}:$pack_id $key_"
|
||||||
|
if isnonnull "$r_pack_delete"
|
||||||
then
|
then
|
||||||
append_to @Packlist "pack :${Hashtype}:$pack_id $key_"
|
append_to @Keeplist "keep :${Hashtype}:$pack_id 1"
|
||||||
fi
|
fi
|
||||||
# else, repack rewrote Packlist
|
|
||||||
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Generate manifest
|
# Generate manifest
|
||||||
|
|
@ -799,20 +767,21 @@ $Extnlist
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Upload pack
|
# Upload pack
|
||||||
if [ -s "$TmpObjlist" ]
|
if [ -s "$tmp_objlist" ]
|
||||||
then
|
then
|
||||||
PUT "$URL" "$pack_id" "$TmpPack_Encrypted"
|
PUT "$URL" "$pack_id" "$tmp_encrypted"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Upload manifest
|
# Upload manifest
|
||||||
PUT "$URL" "$Manifestfile" "$TmpManifest_Enc"
|
PUT "$URL" "$Manifestfile" "$TmpManifest_Enc"
|
||||||
|
|
||||||
rm -f "$TmpPack_Encrypted"
|
rm -f "$tmp_encrypted"
|
||||||
rm -f "$TmpObjlist"
|
rm -f "$tmp_objlist"
|
||||||
rm -f "$TmpManifest_Enc"
|
rm -f "$TmpManifest_Enc"
|
||||||
|
|
||||||
# Delete packs
|
# Delete packs
|
||||||
if isnonnull "$r_pack_delete"; then
|
if isnonnull "$r_pack_delete"; then
|
||||||
|
rm -r -f "$Localdir/pack"
|
||||||
REMOVE "$URL" "$(xecho "$r_pack_delete" | while read packline_
|
REMOVE "$URL" "$(xecho "$r_pack_delete" | while read packline_
|
||||||
do
|
do
|
||||||
isnonnull "$packline_" || continue
|
isnonnull "$packline_" || continue
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue