- Engine partially rewrited,

- Full processing now multithreaded,
- Better dup support, tested,
- Purge (KO & Dup) should be easilly implemented,
- Statistics re-implemented, to be tested,
- ROM listings should be re enabled.
This commit is contained in:
agibert 2009-03-25 08:46:41 +00:00
parent c28184b267
commit 386c71c071

740
nds2rm
View File

@ -5,9 +5,9 @@
# (C) 2009 Arnaud G. Gibert
#-----------------------------------------------------------------------------------------------------------------------------------
# $RCSfile: nds2rm,v $
# $Revision: 1.5 $
# $Revision: 1.6 $
# $Name: $
# $Date: 2009/03/24 09:44:37 $
# $Date: 2009/03/25 08:46:41 $
# $Author: agibert $
#-----------------------------------------------------------------------------------------------------------------------------------
@ -32,7 +32,7 @@
#NDS_REPOSITORY_IN=/opt/public/nds
#NDS_REPOSITORY_IN=/opt/public/nds.new
NDS_REPOSITORY_IN=/opt/public/nds.new
#NDS_REPOSITORY_IN=/opt/public/nds.new/nds.old
#NDS_REPOSITORY_IN=/opt/public/nds.new/nds.new
NDS_REPOSITORY_OUT=/opt/public/nds.new
@ -41,12 +41,18 @@ NDS_ROMS_IN=${NDS_REPOSITORY_IN}/roms
NDS_ROMS_OUT=${NDS_REPOSITORY_OUT}/roms
NDS_DAT=${NDS_REPOSITORY_IN}/misc/dsrom.dat
STATS_FILE="stats.txt"
TMP_DIR=$(mktemp -d)
JOB_NB=8
NDS_VERSION="$Name: $"
RNFH_REGEX=".... - .* \(.*:.*\) \[..\] \{.*\} <.*>[-]*[0-9]*\.zip$"
@ -130,6 +136,55 @@ function args_parse()
# Load Dat File
#-----------------------------------------------------------------------------------------------------------------------------------
function dat_load()
{
echo "loading dsrom.dat..."
IFS=';'
while read line
do
set ${line}
idx=${2/*(0)/}
if [[ "${idx}" != "xxxx" ]]
then
echo -ne "$2"
if [[ "${verbose}" == "yes" ]]
then
echo
else
printf "\r"
fi
dat_rn[${idx}]="${1}"
dat_id[${idx}]="${2}"
dat_title[${idx}]="${3}"
dat_region[${idx}]="${4}"
dat_lang[${idx}]="${5}"
dat_save_type[${idx}]="${6}"
dat_crc[${idx}]="${7}"
dat_fp[${idx}]="????"
dat_status[${idx}]="?"
dat_dup[${idx}]="0"
dat_filename[${idx}]=""
dat_pid[${idx}]="0"
loaded_cnt=$((${loaded_cnt} + 1))
fi
done < ${NDS_DAT}
}
# Initialize Job Control
#-----------------------------------------------------------------------------------------------------------------------------------
@ -254,7 +309,7 @@ function rom_list_dump()
do
if [[ ${dat_status[${i}]} == "${status}" ]]
then
echo "${dat_id[${i}]} - ${dat_title[${i}]} (${dat_region[${i}]}) [${dat_lang[${i}]}] {${dat_crc[${i}]}} <${dat_fp[${i}]}>"
echo "${dat_id[${i}]} - ${dat_title[${i}]} (${dat_region[${i}]}:${dat_lang[${i}]}) {${dat_crc[${i}]}} <${dat_fp[${i}]}>"
fi
i=$(( ${i} + 1 ))
@ -303,7 +358,7 @@ function dsrom_dump()
then
mask=$(mask_get "${dat_save_type[${i}]}")
echo "${dat_id[${i}]} ${dat_fp[${i}]}-0 $mask ${dat_title[${i}]} (${dat_region[${i}]}) [${dat_lang[${i}]}] {${dat_crc[${i}]}}"
echo "${dat_id[${i}]} ${dat_fp[${i}]}-0 $mask ${dat_title[${i}]} (${dat_region[${i}]}:${dat_lang[${i}]}) [${dat_status[${i}]}] {${dat_crc[${i}]}}"
fi
i=$(( ${i} + 1 ))
@ -362,78 +417,23 @@ function missing_dump()
# Update Statistics
#-----------------------------------------------------------------------------------------------------------------------------------
function stats_update()
{
action=${1}
mode=${2}
case ${action} in
("set_status")
if [[ ${status} == "OK" ]]
then
ss_ok_cnt=$((${ss_ok_cnt} + 1))
else
ss_ko_cnt=$((${ss_ko_cnt} + 1))
fi
;;
("change_status")
if [[ ${status} == "OK" ]]
then
cs_ok_cnt=$((${cs_ok_cnt} + 1))
else
cs_ko_cnt=$((${cs_ko_cnt} + 1))
fi
;;
("keep_status")
if [[ ${status} == "OK" ]]
then
ks_ok_cnt=$((${ks_ok_cnt} + 1))
else
ks_ko_cnt=$((${ks_ko_cnt} + 1))
fi
;;
*)
echo "stats_update internal error [$1] !";
exit 1
;;
esac
ok_cnt=$((${ss_ok_cnt} + ${cs_ok_cnt} + ${ks_ok_cnt}))
ko_cnt=$((${ss_ko_cnt} + ${cs_ko_cnt} + ${ks_ko_cnt}))
found_cnt=$((${ok_cnt} + ${ko_cnt}))
missing_cnt=$((${missing_cnt} + ${idx} - ${idx_old} - 1))
idx_old=${idx}
}
# Make Arc FileName
#-----------------------------------------------------------------------------------------------------------------------------------
function arc_mkfn()
{
file_name_ptr=${1}
dir=${2}
id=${3}
title=${4}
reg=${5}
lang=${6}
crc=${7}
fp=${8}
status=${9}
dup_id=${10}
id=${2}
title=${3}
reg=${4}
lang=${5}
crc=${6}
fp=${7}
status=${8}
dup_id=${9}
file_prefix="${dir}/${id} - ${title} (${reg}) [${lang}] {${crc}} <${fp}> ${status}"
file_prefix="${id} - ${title} (${reg}:${lang}) [${status}] {${crc}} <${fp}>"
if [[ ${dup_id} == "" ]]
then
@ -472,10 +472,12 @@ function arc_build()
print_verbose "zipping ${rom} into ${file_name}"
zip -m9 "${file_name}" "${rom}" >/dev/null 2>&1 &
# zip -m9 "${file_name}" "${rom}" >/dev/null 2>&1 &
job_pid[${job_id}]=$!
dat_pid[${idx}]=$!
# job_pid[${job_id}]=$!
# dat_pid[${idx}]=$!
zip -m9 "${NDS_ROMS_OUT}/${file_name}" "${rom}" >/dev/null 2>&1
else
print_verbose "removing ${rom}"
@ -506,11 +508,11 @@ function arc_rename()
then
print_verbose "moving ${file} to ${file_name}"
mv "${NDS_ROMS_IN}/${file}" "${file_name}"
mv "${NDS_ROMS_IN}/${file}" "${NDS_ROMS_OUT}/${file_name}"
else
print_verbose "copying ${file} to ${file_name}"
cp "${NDS_ROMS_IN}/${file}" "${file_name}"
cp "${NDS_ROMS_IN}/${file}" "${NDS_ROMS_OUT}/${file_name}"
fi
fi
fi
@ -523,21 +525,15 @@ function arc_rename()
# Move Duplicate
#-----------------------------------------------------------------------------------------------------------------------------------
function dup_mv()
function arc_mv()
{
file_name_trg=${1}
file_name_src=${2}
mode=${3}
pid=${4}
if [[ "${mode}" == "correct" ]]
then
if [[ "${pid}" != "0" ]]
then
wait "${pid}"
fi
print_verbose "moving ${file_name_src} to ${file_name_trg}"
mv "${file_name_src}" "${file_name_trg}"
@ -619,90 +615,14 @@ function tmp_mv()
# DSROM Mode
#-----------------------------------------------------------------------------------------------------------------------------------
function dsrom_mode()
{
# Remove .tmp suffix
tmp_mv
echo "scanning rom repository..."
IFS=' '
tmp_file=$(mktemp)
ls ${NDS_ROMS_IN} > ${tmp_file}
while read file
do
set ${file:0:4}
id=$1
idx=${id/*(0)/}
# Get info from dat
title=${dat_title[${idx}]}
reg=${dat_region[${idx}]}
lang=${dat_lang[${idx}]}
crc=${dat_crc[${idx}]}
regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..[-]*[0-9]*\.zip$"
# Type of archive ?
if [[ ${file} =~ $regex ]]
then
fp=${file/*\} \</}; fp=${fp/\> */}
status=${file/* /}; status=${status/.zip/}
dat_fp[${idx}]=${fp}
dat_status[${idx}]=${status}
fi
done < ${tmp_file}
\rm -f ${tmp_file}
dsrom_dump
}
# Test and Correct Mode
#-----------------------------------------------------------------------------------------------------------------------------------
function test_correct_mode()
function arc_test_correct()
{
if [[ ${in_place} == "yes" ]]
then
file_suffix=".zip.tmp"
else
file_suffix=".zip"
fi
file=${1}
found_cnt=${2}
# Initialize Job Control
job_init
# Remove .tmp suffix
tmp_mv
echo "scanning rom repository..."
IFS=' '
tmp_file=$(mktemp)
idx_old=0
ls ${NDS_ROMS_IN} > ${tmp_file}
while read file
do
job_switch "${job_id}"
set ${file:0:4}
@ -716,11 +636,9 @@ function test_correct_mode()
lang=${dat_lang[${idx}]}
crc=${dat_crc[${idx}]}
regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..[-]*[0-9]*\.zip$"
# Type of archive ?
if [[ ${file} =~ $regex ]]
if [[ ${file} =~ ${RNFH_REGEX} ]]
then
arc_type="rx3"
else
@ -732,7 +650,7 @@ function test_correct_mode()
if [[ ( "${arc_type}" != "rx3" ) || ( "${rebuild}" == "yes" ) ]]
then
rom="${id} - ${title} (${reg}) [${lang}].nds"
rom="${id} - ${title} (${reg}:${lang}).nds"
rom_extract "${rom}" "${file}"
@ -740,7 +658,7 @@ function test_correct_mode()
fp=$(dd skip=12 count=4 bs=1 <"${rom}" 2>/dev/null)
else
crc2=${file/*\{/}; crc2=${crc2/\}*/}
fp=${file/*\} \</}; fp=${fp/\> */}
fp=${file/*\} \</}; fp=${fp/\>*/}
fi
@ -748,7 +666,7 @@ function test_correct_mode()
if [[ "${arc_type}" == "rx3" ]]
then
status=${file/* /}; status=${status/.zip/}
status=${file/* \[/}; status=${status/\] {*/}
if [[ "${crc}" != "${crc2}" ]]
then
@ -782,6 +700,7 @@ function test_correct_mode()
# Try to Renumber if KO
renumed="no"
if [[ ( ${status} == "KO" ) && ( "${renum}" == "yes" ) ]]
then
@ -795,9 +714,10 @@ function test_correct_mode()
reg=${dat_region[${idx}]}
lang=${dat_lang[${idx}]}
crc=${dat_crc[${idx}]}
status2="OK"
# Rename the ROM file
rom2="${id} - ${title} (${reg}) [${lang}].nds"
rom2="${id} - ${title} (${reg}:${lang}) [${status2}].nds"
print_verbose "moving ${rom} to ${rom2}"
@ -806,55 +726,43 @@ function test_correct_mode()
if [[ "${arc_type}" == "rx3" ]]
then
status=${file/* /}; status=${status/.zip/}
status=${file/* \[/}; status=${status/\] {*/}
if [[ "${status}" == "OK" ]]
if [[ "${status}" == "${status2}" ]]
then
action="keep_status"
else
action="change_status"
status="OK"
status=${status2}
fi
else
action="set_status"
status="OK"
status=${status2}
fi
renum_cnt=$((${renum_cnt} + 1))
renumed="yes"
fi
fi
# Duplicate handling & Update Dat Info
if [[ "${dat_status[${idx}]}" != "?" ]]
then
dat_dup[${idx}]=$((${dat_dup[${idx}]} + 1))
dup_id="${dat_dup[${idx}]}"
dup_cnt=$((${dup_cnt} + 1))
if [[ ( "${dat_status[${idx}]}" == "KO" ) && ( "${status}" == "OK" ) ]]
then
dup_file_name="${dat_filename[${idx}]/${file_suffix}/-${dup_id}${file_suffix}}"
dup_mv "${dup_file_name}" "${dat_filename[${idx}]}" "${mode}" "${dat_pid[${idx}]}"
arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" ""
dat_fp[${idx}]=${fp}
dat_status[${idx}]=${status}
dat_filename[${idx}]=${file_name}
fi
arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" "${dup_id}"
else
arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" ""
dat_fp[${idx}]=${fp}
dat_status[${idx}]=${status}
dat_filename[${idx}]=${file_name}
# Rename ROM if needed
if [[ ( "${arc_type}" != "rx3" ) || ( "${rebuild}" == "yes" ) ]]
then
rom2="${id} - ${title} (${reg}:${lang}) [${status}].nds"
print_verbose "moving ${rom} to ${rom2}"
mv "${rom}" "${rom2}"
rom=${rom2}
fi
fi
# Make File Name
arc_mkfn "file_name" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" "${found_cnt}"
@ -869,13 +777,160 @@ function test_correct_mode()
# Updates Stats
echo "${renumed}" "${action}" "${status}" >"${STATS_FILE}"
#stats_update "${action}" "${mode}"
}
# Update Statistics
#-----------------------------------------------------------------------------------------------------------------------------------
function stats_update()
{
renumed=${1}
action=${2}
status=${3}
mode=${4}
case ${action} in
("set_status")
if [[ ${status} == "OK" ]]
then
ss_ok_cnt=$((${ss_ok_cnt} + 1))
else
ss_ko_cnt=$((${ss_ko_cnt} + 1))
fi
;;
("change_status")
if [[ ${status} == "OK" ]]
then
cs_ok_cnt=$((${cs_ok_cnt} + 1))
else
cs_ko_cnt=$((${cs_ko_cnt} + 1))
fi
;;
("keep_status")
if [[ ${status} == "OK" ]]
then
ks_ok_cnt=$((${ks_ok_cnt} + 1))
else
ks_ko_cnt=$((${ks_ko_cnt} + 1))
fi
;;
*)
echo "stats_update internal error [$1] !";
exit 1
;;
esac
}
# Proceed Stats
#-----------------------------------------------------------------------------------------------------------------------------------
function stats_proceed()
{
ss_ok_cnt=0
ss_ko_cnt=0
cs_ok_cnt=0
cs_ko_cnt=0
ks_ok_cnt=0
ks_ko_cnt=0
while read file
do
set ${file}
renumed=${1}
action=${2}
status=${3}
stats_update "${renumed}" "${action}" "${status}"
done < "${STATS_FILE}"
\rm -f "${STATS_FILE}"
found_cnt=$((${ok_cnt} + ${ko_cnt}))
missing_cnt=$((${loaded_cnt} - ${found_cnt}))
}
# Print Stats
#-----------------------------------------------------------------------------------------------------------------------------------
function stats_print()
{
printf "%4d roms set to OK\n" ${ss_ok_cnt}
printf "%4d roms changed to OK\n" ${cs_ok_cnt}
printf "%4d roms kept to OK\n" ${ks_ok_cnt}
printf "%4d roms are OK\n" ${ok_cnt}
echo
printf "%4d roms set to KO\n" ${ss_ko_cnt}
printf "%4d roms changed to KO\n" ${cs_ko_cnt}
printf "%4d roms kept to KO\n" ${ks_ko_cnt}
printf "%4d roms are KO\n" ${ko_cnt}
echo
printf "%4d roms are in dsrom.dat\n" ${loaded_cnt}
printf "%4d roms are found\n" ${found_cnt}
printf "%4d roms are renumbered\n" ${renum_cnt}
printf "%4d roms are duplicated\n" ${dup_cnt}
printf "%4d roms are missing\n" ${missing_cnt}
}
# Manage Archives
#-----------------------------------------------------------------------------------------------------------------------------------
function arc_mng()
{
# Initialize Job Control
job_init
echo "processing rom repository..."
IFS=' '
found_cnt=0
tmp_file=$(mktemp)
ls ${NDS_ROMS_IN} > ${tmp_file}
while read file
do
job_switch "${job_id}"
arc_test_correct "${file}" "${found_cnt}" &
job_pid[${job_id}]=$!
# Update stats
stats_update "${action}" "${mode}"
# stats_update "${action}" "${mode}"
found_cnt=$((${found_cnt} + 1))
printf "found: %4d OK: %4d KO: %4d renumbered: %4d duplicated: %4d missing: %4d" ${found_cnt} ${ok_cnt} ${ko_cnt} ${renum_cnt} ${dup_cnt} ${missing_cnt}
# printf "found: %4d OK: %4d KO: %4d renumbered: %4d duplicated: %4d missing: %4d" ${found_cnt} ${ok_cnt} ${ko_cnt} ${renum_cnt} ${dup_cnt} ${missing_cnt}
printf "found: %4d" ${found_cnt}
if [[ "${verbose}" == "yes" ]]
then
@ -884,74 +939,199 @@ function test_correct_mode()
printf "\r"
fi
done < ${tmp_file}
done < ${tmp_file}
\rm -f ${tmp_file}
\rm -f ${tmp_file}
wait
wait
if [[ "${verbose}" != "yes" ]]
then
echo " "
fi
echo
# Remove .tmp suffix
#-----------------------------------------------------------------------------------------------------------------------------------
if [[ ${in_place} == "yes" ]]
if [[ "${verbose}" != "yes" ]]
then
tmp_mv
echo " "
fi
# Initialize Job Control
echo
# Collect Stats"
cd "${TMP_DIR}"
cat *"/${STATS_FILE}" > "${STATS_FILE}"
# DeInitialize Job Control
job_deinit
}
# Dump ROMs List
# Manage Arc Suffix
#-----------------------------------------------------------------------------------------------------------------------------------
if [[ ${lok} == "yes" ]]
then
ok_dump
fi
function suffix_mng()
{
echo "managing duplicate archives..."
if [[ ${lko} == "yes" ]]
then
ko_dump
fi
IFS=' '
found_cnt=0
ok_cnt=0
ko_cnt=0
dup_cnt=0
idx_old=0
if [[ ${lmiss} == "yes" ]]
then
missing_dump
fi
tmp_file=$(mktemp)
ls ${NDS_ROMS_OUT} > ${tmp_file}
while read file
do
set ${file:0:4}
id=$1
idx=${id/*(0)/}
status=${file/* \[/}; status=${status/\] \{*/}
crc_fp=${file/*\] \{/{}; crc_fp=${crc_fp/-*/}
if [[ ${idx} != ${idx_old} ]]
then
idx_old=${idx}
dup_id=0
file_name="${file/) \[*/}) [${status}] ${crc_fp}.zip"
if [[ "${status}" == "OK" ]]
then
ok_cnt=$((${ok_cnt} + 1))
else
ko_cnt=$((${ko_cnt} + 1))
fi
else
dup_id=$((${dup_id} + 1))
file_name="${file/) \[*/}) [${status}] ${crc_fp}-${dup_id}.zip"
dup_cnt=$((${dup_cnt} + 1))
fi
print_verbose "moving ${file} to ${file_name}"
arc_mv "${NDS_ROMS_OUT}/${file_name}" "${NDS_ROMS_OUT}/${file}" "${mode}"
# Print statistics
found_cnt=$((${found_cnt} + 1))
printf "found: %4d" ${found_cnt}
if [[ "${verbose}" == "yes" ]]
then
echo
else
printf "\r"
fi
done < ${tmp_file}
\rm -f ${tmp_file}
}
# DSROM Mode
#-----------------------------------------------------------------------------------------------------------------------------------
missing_cnt=$((${loaded_cnt} - ${found_cnt}))
function dsrom_mode()
{
echo "scanning rom repository..."
IFS=' '
printf "%4d roms set to OK\n" ${ss_ok_cnt}
printf "%4d roms changed to OK\n" ${cs_ok_cnt}
printf "%4d roms kept to OK\n" ${ks_ok_cnt}
printf "%4d roms are OK\n" ${ok_cnt}
echo
printf "%4d roms set to KO\n" ${ss_ko_cnt}
printf "%4d roms changed to KO\n" ${cs_ko_cnt}
printf "%4d roms kept to KO\n" ${ks_ko_cnt}
printf "%4d roms are KO\n" ${ko_cnt}
echo
printf "%4d roms are in dsrom.dat\n" ${loaded_cnt}
printf "%4d roms are found\n" ${found_cnt}
printf "%4d roms are renumbered\n" ${renum_cnt}
printf "%4d roms are duplicated\n" ${dup_cnt}
printf "%4d roms are missing\n" ${missing_cnt}
tmp_file=$(mktemp)
ls ${NDS_ROMS_IN} > ${tmp_file}
while read file
do
set ${file:0:4}
id=$1
idx=${id/*(0)/}
# Get info from dat
title=${dat_title[${idx}]}
reg=${dat_region[${idx}]}
lang=${dat_lang[${idx}]}
crc=${dat_crc[${idx}]}
# Type of archive ?
if [[ ${file} =~ ${RNFH_REGEX} ]]
then
fp=${file/*\} \</}; fp=${fp/\>*/}
status=${file/* \[/}; status=${status/\] \{*/}
dat_fp[${idx}]=${fp}
dat_status[${idx}]=${status}
fi
done < ${tmp_file}
\rm -f ${tmp_file}
# Dump it !
dsrom_dump
}
# Test and Correct Mode
#-----------------------------------------------------------------------------------------------------------------------------------
function test_correct_mode()
{
if [[ ${in_place} == "yes" ]]
then
file_suffix=".zip.tmp"
else
file_suffix=".zip"
fi
# Remove .tmp suffix
tmp_mv
# Process Archives
arc_mng
# Suffix Management
suffix_mng
# Dump ROMs List
#-----------------------------------------------------------------------------------------------------------------------------------
# if [[ ${lok} == "yes" ]]
# then
# ok_dump
# fi
#
# if [[ ${lko} == "yes" ]]
# then
# ko_dump
# fi
#
# if [[ ${lmiss} == "yes" ]]
# then
# missing_dump
# fi
# Collect and Proceed Stats
stats_proceed
# Print statistics
stats_print
}
@ -961,20 +1141,15 @@ printf "%4d roms are missing\n" ${missing_cnt}
# Main
#-----------------------------------------------------------------------------------------------------------------------------------
cd ${TMP_DIR}
# Initialise stats variables
loaded_cnt=0
ss_ok_cnt=0
ss_ko_cnt=0
cs_ok_cnt=0
ct_ko_cnt=0
ks_ok_cnt=0
ks_ko_cnt=0
ok_cnt=0
ko_cnt=0
found_cnt=0
missing_cnt=0
dup_cnt=0
renum_cnt=0
@ -1005,44 +1180,7 @@ fi
#-----------------------------------------------------------------------------------------------------------------------------------
echo "loading dsrom.dat..."
IFS=';'
while read line
do
set ${line}
idx=${2/*(0)/}
if [[ "${idx}" != "xxxx" ]]
then
echo -ne "$2"
if [[ "${verbose}" == "yes" ]]
then
echo
else
printf "\r"
fi
dat_rn[${idx}]="${1}"
dat_id[${idx}]="${2}"
dat_title[${idx}]="${3}"
dat_region[${idx}]="${4}"
dat_lang[${idx}]="${5}"
dat_save_type[${idx}]="${6}"
dat_crc[${idx}]="${7}"
dat_fp[${idx}]="????"
dat_status[${idx}]="?"
dat_dup[${idx}]="0"
dat_filename[${idx}]=""
dat_pid[${idx}]="0"
loaded_cnt=$((${loaded_cnt} + 1))
fi
done < ${NDS_DAT}
dat_load
@ -1056,7 +1194,13 @@ else
test_correct_mode
fi
exit 0
#-----------------------------------------------------------------------------------------------------------------------------------
# Final Cleanup
cd /tmp
rmdir ${TMP_DIR}
exit 0