MozillaQualityAssurance:Web Page Set

#! /bin/bash
#takes two inputs, $1 = file containing list of web pages of form http://pagename
#                  $2 = output file where list of index files is dumped - useful for places list of links into scripts
#
# web pages are dropped in directories named for their urls
if [ $# != 2 ]; then
        echo 'missing command line arguments'
        echo
        echo 'usage: getpages.sh inputfile outputfile'
        echo '  inputfile: file containing one url per line of the form http://url'
        echo '  outputfile: file to be created during execution, contains a list of index files one per url'
        exit
fi
for URL in $(cat $1); do
        #strip the leading http:// from the url
        CLEANURL=$(echo $URL | sed -e 's/http:\/\/\(.*\)/\1/')
        #create a directory with the cleaned url as the name
        mkdir $CLEANURL
        cd $CLEANURL
        wget -p -k -H -E -erobots=off --no-check-certificate -U "Mozilla/5.0 (firefox)" --restrict-file-names=windows $URL -o outputlog.txt
        #figure out where/what the index file for this page is from the wget output log
        FILENAME=$(grep "saved" outputlog.txt | head -1 | sed -e "s/.*\`\(.*\)\'.*/\1/")
        rm outputlog.txt
        cd ..
        #add the index file link to the list of index files being generated
        echo $CLEANURL/$FILENAME >> $2
done
#call the cleanse.sh script to disable any trailing non-localized urls found in the web pages
./cleanse.sh $1

#!/bin/sh 
if [ $# = 0 ]; then
  echo 'usage: cleanse.sh dir1 [dir2...]'
  exit 1
fi
# generates the list of files to be cleansed (exclude image files)
get_files() {
  find "$1" -type f -a ! -iname \*.jpg -a ! -iname \*.gif -a ! -iname \*.png -a ! -name \*.bak
}
# TODO(darin): make this more performant
cleanse_file() {
  echo "______cleansing $1" 
  perl -pi -e 's/[a-zA-Z0-9_]*.writeln/void/g' $1
  perl -pi -e 's/[a-zA-Z0-9_]*.write/void/g' $1
  perl -pi -e 's/[a-zA-Z0-9_]*.open/void/g' $1
  perl -pi -e 's/"https/"httpsdisabled/gi' $1
  perl -pi -e 's/"http/"httpdisabled/gi' $1
  perl -pi -e 's/<object/<objectdisabled/gi' $1
  perl -pi -e 's/<embed/<embeddisabled/gi' $1
  perl -pi -e 's/load/loaddisabled/g' $1
}
#dirs=($*)
#dirsLen=${#dirs[*]}
for URL in $(cat $1); do
  CLEANURL=$(echo $URL | sed -e 's/http:\/\/\(.*\)/\1/')
  files=$(get_files $CLEANURL)
  for f in $files; do
    cleanse_file $f
  done
done