The shell reads the data table and assigns values to variables

Time:2021-10-26

Download data using shell

#!/bin/bash

HOST="127.0.0.1"
USER="root"
PASSWORD="password"
PORT=3306

MYSQL="mysql -h ${HOST} -u ${USER} -P ${PORT}  -p ${PASSWORD}"

#Method 1
while read -a row
do
    echo "${row[0]} == ${row[1]}"
done< <(echo "select id,url from table_name limit 30;" | ${MYSQL})

while read ID URL
do
    echo "${ID} == ${URL}"
done< <(echo "select id,url from table_name limit 30;" | ${MYSQL})

String MD5

#!/bin/bash
echo -n $1|md5sum|cut -d ' ' -f1

Get proxy IP information

#!/bin/bash
MYSQL="mysql -h127.0.0.1 -uroot -ppassword -P3306"
sql="select addres from table_name"
result="$($MYSQL -e "$sql")"
dump_data=./ip.log > $dump_data
echo -e "$result" > $dump_data
#Note that echo - e "$result" > $dump_ Data must be enclosed in double quotation marks to prevent the exported data from being crowded in one line

Download shell

#!/bin/bash

LOG_FILE="down.log";
STATE_INFO=""
PASSWORDSS='password'
#MYSQL=`which mysql`
MYSQL="mysql -h127.0.0.1 -uroot -p${PASSWORDSS} -P3306"

if [[ $1 -lt 1 ]]; then
    Echo "please enter the starting value of ID"; exit -1;
fi

if [[ $2 -lt 1 ]]; then
    Echo "please enter ID end value"; exit -1;
fi

#Get proxy IP
PROXY_LIST=[]
j=0
while read line
do
    PROXY_LIST[j]=${line}
    j=$(($j+1))
done < ip.log
PROXY_NUM=${#PROXY_LIST[@]}

#Cyclic download data
while read ID URL
do
    if [[ $ID -gt 0 ]]; then
        #Get file path
        PATH_ID=$(printf "%08d" ${ID})
        PATH_0=${PATH_ID:0:2}
        PATH_1=${PATH_ID:2:2}
        PATH_2=${PATH_ID:4:2}

        #Get original file extension
        FILE_NAME=$(basename "${URL}")
        EXTENSION="${FILE_NAME##*.}"

        #Get MD5 file name
        PIC_MD5=$(source ./md5.sh ${PATH_ID})

        #Save directory
        SAVE_PATH_DIR="pic/${PATH_0}/${PATH_1}/${PATH_2}/"

        #Create folder
        mkdir -p ${SAVE_PATH_DIR}

        #New file name
        NEW_FILE_NAME="${ID}-${PIC_MD5}.${EXTENSION}"

        #Database save name
        DB_PATH_DIR="${PATH_0}/${PATH_1}/${PATH_2}/${NEW_FILE_NAME}"

        #The proxy IP is obtained
        PROXY_KEY=$(($RANDOM%${PROXY_NUM}))
        PROXY=${PROXY_LIST[${PROXY_KEY}]}

        #Download pictures and save them locally
        wget -q -e "http_proxy=http://${PROXY}" -O ${SAVE_PATH_DIR}${NEW_FILE_NAME} ${URL} --no-check-certificate

        #Download successfully updated database information
        if [[ $? -eq 0 ]]; then
            #Update database
            echo "update table_name set url='${DB_PATH_DIR}' where id=${ID};" | ${MYSQL}

            STATE_INFO="Success"
         else
            STATE_INFO="Fail"
        fi

        #Save log information
        echo "ID:${ID} ${STATE_INFO}"
        echo $(date +%Y"."%m"."%d" "%k":"%M":"%S) ${ID} ${STATE_INFO} >> ${LOG_FILE}
    fi
done< <(echo "select id,url from table_name where id between $1 and $2 order by id asc" | ${MYSQL})