- New duplicated ROM management system,

- Add job control system.
This is the first optimized release.
This commit is contained in:
agibert 2009-03-23 19:55:57 +00:00
parent 48d1f42f3b
commit 673a21acdb

376
nds2rm
View File

@ -5,9 +5,9 @@
# (C) 2009 Arnaud G. Gibert
#-----------------------------------------------------------------------------------------------------------------------------------
# $RCSfile: nds2rm,v $
# $Revision: 1.3 $
# $Revision: 1.4 $
# $Name: $
# $Date: 2009/03/22 22:31:58 $
# $Date: 2009/03/23 19:55:57 $
# $Author: agibert $
#-----------------------------------------------------------------------------------------------------------------------------------
@ -41,6 +41,10 @@ NDS_ROMS_IN=${NDS_REPOSITORY_IN}/roms
NDS_ROMS_OUT=${NDS_REPOSITORY_OUT}/roms
NDS_DAT=${NDS_REPOSITORY_IN}/misc/dsrom.dat
TMP_DIR=$(mktemp -d)
JOB_NB=8
NDS_VERSION="$Name: $"
@ -106,7 +110,7 @@ function args_parse()
--list_miss) lmiss="yes"; shift;;
-v|--verbose) verbose="yes"; shift;;
--) shift; break;;
*) echo "Internal error [$1] !"; exit 1;;
*) echo "args_parse internal error [$1] !"; exit 1;;
esac
done
}
@ -115,6 +119,71 @@ function args_parse()
# Initialize Job Control
#-----------------------------------------------------------------------------------------------------------------------------------
function job_init()
{
job_id=0
while [[ ${job_id} -lt ${JOB_NB} ]]
do
job_pid[${job_id}]="0"
job_dir[${job_id}]="${TMP_DIR}/nds2rom-$$-${job_id}"
mkdir "${job_dir[${job_id}]}"
job_id=$((${job_id} + 1))
done
}
# DeInitialize Job Control
#-----------------------------------------------------------------------------------------------------------------------------------
function job_deinit()
{
job_id=0
while [[ ${job_id} -lt ${JOB_NB} ]]
do
rm -R "${job_dir[${job_id}]}"
job_id=$((${job_id} + 1))
done
}
# Switch Job
#-----------------------------------------------------------------------------------------------------------------------------------
function job_switch()
{
job_id=$((${job_id} + 1))
if [[ ${job_id} -ge ${JOB_NB} ]]
then
job_id=0
fi
cd "${job_dir[${job_id}]}"
if [[ "${job_pid[${job_id}]}" != "0" ]]
then
wait "${job_pid[${job_id}]}"
fi
}
# Lookup CRC
#-----------------------------------------------------------------------------------------------------------------------------------
@ -320,7 +389,7 @@ function stats_update()
;;
*)
echo "Internal error [$1] !";
echo "stats_update internal error [$1] !";
exit 1
;;
esac
@ -336,41 +405,10 @@ function stats_update()
# Lookup Duplicate Arc
# Make Arc FileName
#-----------------------------------------------------------------------------------------------------------------------------------
function arc_dup_lookup()
{
dup_name_ptr=${1}
file_prefix=${2}
file_suffix=${3}
if [[ -f "${file_prefix}${file_suffix}" ]]
then
i=1
while [[ -f "${file_prefix}-${i}.{file_suffix}" ]]
do
i=$((${i} + 1))
done
dup_name="-${i}"
dup_cnt=$((${dup_cnt} + 1))
else
dup_name=""
fi
eval ${dup_name_ptr}=${dup_name}
}
# Add Arc
#-----------------------------------------------------------------------------------------------------------------------------------
function arc_add()
function arc_mkfn()
{
file_name_ptr=${1}
dir=${2}
@ -381,25 +419,16 @@ function arc_add()
crc=${7}
fp=${8}
status=${9}
dup_id=${10}
file_prefix="${dir}/${id} - ${title} (${reg}) [${lang}] {${crc}} <${fp}> ${status}"
file_suffix=".zip"
if [[ -f "${file_prefix}${file_suffix}" ]]
if [[ ${dup_id} == "" ]]
then
arc_dup_lookup "dup_name" "${file_prefix}" "${file_suffix}"
if [[ "${status}" == "OK" ]]
then
print_verbose "moving ${file_prefix}${file_suffix} to ${file_prefix}${dup_name}${file_suffix}"
mv "${file_prefix}${file_suffix}" "${file_prefix}${dup_name}${file_suffix}"
dup_name=""
fi
else
dup_name=""
else
dup_name="-${dup_id}"
fi
eval ${file_name_ptr}=\"${file_prefix}${dup_name}${file_suffix}\"
@ -409,51 +438,16 @@ function arc_add()
# Remove Arc
#-----------------------------------------------------------------------------------------------------------------------------------
#function arc_remove()
#{
# file=${1}
#
#
# prefix=${file/\> ??*.zip/>}
#
# arc_list=$(mktemp)
# ls ${prefix}* > ${arc_list}
#
# arc_mode="scan"
#
# while read arc
# do
# case "${arc_mode}"
# in
#
# esac
# done < ${arc_list}
#
# \rm -f ${arc_list}
#}
# Build Arc
#-----------------------------------------------------------------------------------------------------------------------------------
function arc_build()
{
id=${1}
title=${2}
reg=${3}
lang=${4}
crc=${5}
fp=${6}
status=${7}
rom=${8}
file=${9}
mode=${10}
file_name=${1}
rom=${2}
file=${3}
mode=${4}
idx=${5}
if [[ "${mode}" == "correct" ]]
@ -463,15 +457,14 @@ function arc_build()
print_verbose "removing ${file}"
\rm -f "${NDS_ROMS_IN}/${file}"
# arc_remove "${file}"
fi
arc_add "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc}" "${fp}" "${status}"
print_verbose "zipping ${rom} into ${file_name}"
zip -m9 "${file_name}" "${rom}" >/dev/null 2>&1
zip -m9 "${file_name}" "${rom}" >/dev/null 2>&1 &
job_pid[${job_id}]=$!
dat_pid[${idx}]=$!
else
print_verbose "removing ${rom}"
@ -488,24 +481,16 @@ function arc_build()
function arc_rename()
{
id=${1}
title=${2}
reg=${3}
lang=${4}
crc=${5}
fp=${6}
status=${7}
file=${8}
action=${9}
mode=${10}
file_name=${1}
file=${2}
action=${3}
mode=${4}
if [[ "${mode}" == "correct" ]]
then
if [[ ${action} != "keep_status" ]]
then
arc_add "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc}" "${fp}" "${status}"
if [[ ${in_place} == "yes" ]]
then
print_verbose "moving ${file} to ${file_name}"
@ -524,6 +509,34 @@ function arc_rename()
# Move Duplicate
#-----------------------------------------------------------------------------------------------------------------------------------
function dup_mv()
{
file_name_trg=${1}
file_name_src=${2}
mode=${3}
pid=${4}
if [[ "${mode}" == "correct" ]]
then
if [[ "${pid}" != "0" ]]
then
wait "${pid}"
fi
print_verbose "moving ${file_name_src} to ${file_name_trg}"
mv "${file_name_src}" "${file_name_trg}"
fi
}
# Extract ROM
#-----------------------------------------------------------------------------------------------------------------------------------
@ -554,7 +567,7 @@ function rom_extract()
;;
(*)
echo "Internal error [$1] !";
echo "rom_extract internal error [$1] !";
exit 1
;;
esac
@ -564,11 +577,47 @@ function rom_extract()
# Remove .tmp suffix
#-----------------------------------------------------------------------------------------------------------------------------------
function tmp_mv()
{
echo "removing .tmp suffix..."
IFS=' '
mv_cnt=0
for file_src in ${NDS_ROMS_OUT}/*.tmp
do
if [[ "${file_src}" != "${NDS_ROMS_OUT}/*.tmp" ]]
then
file_trg=${file_src/.zip.tmp/.zip}
print_verbose "move ${file_src} to ${file_trg}"
mv "${file_src}" "${file_trg}"
mv_cnt=$((${mv_cnt} + 1))
printf "moved: %4d\r" "${mv_cnt}"
fi
done
}
# DSROM Mode
#-----------------------------------------------------------------------------------------------------------------------------------
function dsrom_mode()
{
# Remove .tmp suffix
tmp_mv
echo "scanning rom repository..."
IFS=' '
@ -590,7 +639,7 @@ function dsrom_mode()
lang=${dat_lang[${idx}]}
crc=${dat_crc[${idx}]}
regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..\.zip$"
regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..[-]*[0-9]*\.zip$"
# Type of archive ?
@ -619,20 +668,35 @@ function dsrom_mode()
function test_correct_mode()
{
echo "scanning rom repository..."
IFS=' '
if [[ ${in_place} == "yes" ]]
then
file_suffix=".zip.tmp"
else
file_suffix=".zip"
fi
tmp_file=$(mktemp)
idx_old=0
# Initialize Job Control
job_init
ls ${NDS_ROMS_IN} > ${tmp_file}
# Remove .tmp suffix
tmp_mv
while read file
do
set ${file:0:4}
echo "scanning rom repository..."
IFS=' '
id=$1
idx=${id/*(0)/}
tmp_file=$(mktemp)
idx_old=0
ls ${NDS_ROMS_IN} > ${tmp_file}
while read file
do
job_switch "${job_id}"
set ${file:0:4}
id=$1
idx=${id/*(0)/}
# Get info from dat
@ -641,8 +705,7 @@ do
lang=${dat_lang[${idx}]}
crc=${dat_crc[${idx}]}
regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..\.zip$"
regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..[-]*[0-9]*\.zip$"
# Type of archive ?
@ -752,12 +815,34 @@ do
# Update Dat Info
# Duplicate handling & Update Dat Info
if [[ "${dat_status[${idx}]}" != "OK" ]]
if [[ "${dat_status[${idx}]}" != "?" ]]
then
dat_fp[${idx}]=${fp}
dat_status[${idx}]=${status}
dat_dup[${idx}]=$((${dat_dup[${idx}]} + 1))
dup_id="${dat_dup[${idx}]}"
dup_cnt=$((${dup_cnt} + 1))
if [[ ( "${dat_status[${idx}]}" == "KO" ) && ( "${status}" == "OK" ) ]]
then
dup_file_name="${dat_filename[${idx}]/${file_suffix}/-${dup_id}${file_suffix}}"
dup_mv "${dup_file_name}" "${dat_filename[${idx}]}" "${mode}" "${dat_pid[${idx}]}"
arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" ""
dat_fp[${idx}]=${fp}
dat_status[${idx}]=${status}
dat_filename[${idx}]=${file_name}
fi
arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" "${dup_id}"
else
arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" ""
dat_fp[${idx}]=${fp}
dat_status[${idx}]=${status}
dat_filename[${idx}]=${file_name}
fi
@ -766,9 +851,9 @@ do
if [[ ( "${arc_type}" != "rx3" ) || ( ${rebuild} == "yes" ) ]]
then
arc_build "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" "${rom}" "${file}" "${mode}"
arc_build "${file_name}" "${rom}" "${file}" "${mode}" "${idx}"
else
arc_rename "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" "${file}" "${action}" "${mode}"
arc_rename "${file_name}" "${file}" "${action}" "${mode}"
fi
@ -794,10 +879,25 @@ done < ${tmp_file}
wait
if [[ "${verbose}" != "yes" ]]
then
echo " "
fi
if [[ "${verbose}" != "yes" ]]
then
echo " "
fi
echo
# Remove .tmp suffix
#-----------------------------------------------------------------------------------------------------------------------------------
if [[ ${in_place} == "yes" ]]
then
tmp_mv
fi
# Initialize Job Control
job_deinit
@ -906,7 +1006,14 @@ do
if [[ "${idx}" != "xxxx" ]]
then
echo -ne "$2\r"
echo -ne "$2"
if [[ "${verbose}" == "yes" ]]
then
echo
else
printf "\r"
fi
dat_rn[${idx}]="${1}"
dat_id[${idx}]="${2}"
@ -915,8 +1022,12 @@ do
dat_lang[${idx}]="${5}"
dat_save_type[${idx}]="${6}"
dat_crc[${idx}]="${7}"
dat_fp[${idx}]="????"
dat_status[${idx}]="?"
dat_dup[${idx}]="0"
dat_filename[${idx}]=""
dat_pid[${idx}]="0"
loaded_cnt=$((${loaded_cnt} + 1))
fi
@ -925,6 +1036,7 @@ done < ${NDS_DAT}
#-----------------------------------------------------------------------------------------------------------------------------------
if [[ "${mode}" == "dsrom" ]]
then
dsrom_mode