summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xparse_stagit80
1 files changed, 58 insertions, 22 deletions
diff --git a/parse_stagit b/parse_stagit
index 39feda2..be52ee3 100755
--- a/parse_stagit
+++ b/parse_stagit
@@ -12,51 +12,87 @@ fi
FIN="/dev/stdin"
URLBASE="$1"
DEST="$2"
-
SUBPATH="/file"
-
CURL="torify curl -Ls "
-PROTO=${URLBASE%%:\/\/*}
-DIRBASE="$PROTO/${URLBASE##[a-z]*:\/\/}"
-echo "proto: $PROTO"
-echo "dirbase: $DIRBASE"
-READMES="README.html README.txt.html README.md.html readme.html readme.txt.html readme.md.html"
+## func
+cleanup () {
+ rm -f $TMPFILE
+ exit
+}
+## func
+get_repos() {
-repos=$($CURL "${URLBASE}" | xml2tsv | grep "/html/body/div/table/tbody/tr/td/a" \
+repos=$(cat $TMPFILE | grep "/html/body/div/table/tbody/tr/td/a" \
| awk '{print $(NF-1), $NF}' \
| sed -E 's/href=//g;s/ /\|/'\
)
+}
+## func
+get_descr() {
+ reponame=$1
+ echo "reponame: $reponame"
+ ## Get author and description
+ cat "$TMPFILE" | grep -Ei -A 2 "/html/body/div/table/tbody/tr/td/a[[:blank:]]+href=${reponame}/log.html" |\
+ tail -2 | cut -f 2-
+
+}
-for r in $repos; do
- name=$(echo "$r" | cut -d "|" -f 2 )
- link=$(echo "$r" | cut -d "|" -f 1 )
- link="${link%%log.html}"
- baselink=$(printf "%s/%s" $URLBASE $link)
- printf "link: %s\nbaselink: %s\n" $link $baselink 1>&2
- REPODIR="$DEST/$DIRBASE/$link/"
- mkdir -p $REPODIR
+## func
+get_readmes() {
+ LINK=$1
+ DESTDIR=$2
for f in $READMES; do
- printf " trying file %s..." $baselink/$SUBPATH/$f
- $CURL "$baselink/$SUBPATH/$f" > $REPODIR/$f.tmp
- failure=$(xml2tsv < $REPODIR/$f.tmp 2>/dev/null | \
+ printf " trying file %s..." $LINK/$f
+ $CURL "$LINK/$f" > $DESTDIR/$f.tmp
+ failure=$(xml2tsv < $DESTDIR/$f.tmp 2>/dev/null | \
grep -Eaic "^/html/head/title[[:blank:]]+404 Not Found")
echo $failure
if [ "$failure" = 1 ]; then
printf "[FAILED]\n"
else
- xml2tsv < $REPODIR/$f.tmp 2>/dev/null | \
+ xml2tsv < $DESTDIR/$f.tmp 2>/dev/null | \
grep -Eai "/html/body/div/pre/a[[:blank:]]+href=#.*[[:blank:]]+class=line" | \
cut -f 6- | \
- sed -E 's/\\n//g;s/\\t/\t/g;s/\\\\/\\/g' > $REPODIR/$f
+ sed -E 's/\\n//g;s/\\t/\t/g;s/\\\\/\\/g' > $DESTDIR/$f
printf "[OK]\n"
fi
- rm -f $REPODIR/$f.tmp
+ rm -f $DESTDIR/$f.tmp
sleep 1
done
+}
+
+# main loop
+
+PROTO=${URLBASE%%:\/\/*}
+DIRBASE="$PROTO/${URLBASE##[a-z]*:\/\/}"
+echo "proto: $PROTO"
+echo "dirbase: $DIRBASE"
+
+READMES="README.html README.txt.html README.md.html readme.html readme.txt.html readme.md.html"
+TMPFILE="./tmp_$$"
+
+trap cleanup EXIT KILL TERM INT
+
+$CURL "${URLBASE}" | xml2tsv > $TMPFILE
+
+get_repos
+
+for r in $repos; do
+ name=$(echo "$r" | cut -d "|" -f 2 )
+ link=$(echo "$r" | cut -d "|" -f 1 )
+ link="${link%%log.html}"
+ baselink=$(printf "%s/%s" $URLBASE $link)
+ printf "link: %s\nbaselink: %s\n" $link $baselink 1>&2
+ REPODIR="$DEST/$DIRBASE/$link/"
+ mkdir -p $REPODIR
+ get_descr $name > ${REPODIR}/DESCR
+ ## Get READMEs
+ get_readmes "$baselink/$SUBPATH" "$REPODIR"
done
+cleanup