- New duplicated ROM management system,

- Add job control system.
This is the first optimized release.

nds2rm

@@ -5,9 +5,9 @@
 # (C) 2009 Arnaud G. Gibert
 #-----------------------------------------------------------------------------------------------------------------------------------
 # $RCSfile: nds2rm,v $
-# $Revision: 1.3 $
+# $Revision: 1.4 $
 # $Name:  $
-# $Date: 2009/03/22 22:31:58 $
+# $Date: 2009/03/23 19:55:57 $
 # $Author: agibert $
 #-----------------------------------------------------------------------------------------------------------------------------------
 
@@ -41,6 +41,10 @@ NDS_ROMS_IN=${NDS_REPOSITORY_IN}/roms
 NDS_ROMS_OUT=${NDS_REPOSITORY_OUT}/roms
 NDS_DAT=${NDS_REPOSITORY_IN}/misc/dsrom.dat
 
+TMP_DIR=$(mktemp -d)
+
+JOB_NB=8
+
 NDS_VERSION="$Name:  $"
 
 
@@ -106,7 +110,7 @@ function args_parse()
                --list_miss) lmiss="yes";                    shift;;
             -v|--verbose) verbose="yes";                    shift;;
 	    --)                                             shift;  break;;
-	    *) echo "Internal error [$1] !"; exit 1;;
+	    *) echo "args_parse internal error [$1] !"; exit 1;;
 	esac
     done
 }
@@ -115,6 +119,71 @@ function args_parse()
 
 
 
+# Initialize Job Control
+#-----------------------------------------------------------------------------------------------------------------------------------
+
+function job_init()
+{
+    job_id=0
+
+    while [[ ${job_id} -lt ${JOB_NB} ]]
+    do
+      job_pid[${job_id}]="0"
+      job_dir[${job_id}]="${TMP_DIR}/nds2rom-$$-${job_id}"
+
+      mkdir "${job_dir[${job_id}]}"
+
+      job_id=$((${job_id} + 1))
+    done
+}
+
+
+
+
+
+# DeInitialize Job Control
+#-----------------------------------------------------------------------------------------------------------------------------------
+
+function job_deinit()
+{
+    job_id=0
+
+    while [[ ${job_id} -lt ${JOB_NB} ]]
+    do
+      rm -R "${job_dir[${job_id}]}"
+
+      job_id=$((${job_id} + 1))
+    done
+}
+
+
+
+
+
+# Switch Job
+#-----------------------------------------------------------------------------------------------------------------------------------
+
+function job_switch()
+{
+    job_id=$((${job_id} + 1))
+
+    if [[ ${job_id} -ge ${JOB_NB} ]]
+    then
+	job_id=0
+    fi
+
+    cd "${job_dir[${job_id}]}"
+
+    if [[ "${job_pid[${job_id}]}" != "0" ]]
+    then
+	wait "${job_pid[${job_id}]}"
+    fi
+}
+
+
+
+
+
 # Lookup CRC
 #-----------------------------------------------------------------------------------------------------------------------------------
 
@@ -320,7 +389,7 @@ function stats_update()
 	;;
 
 	*)
-	    echo "Internal error [$1] !";
+	    echo "stats_update internal error [$1] !";
 	    exit 1
 	;;
     esac
@@ -336,41 +405,10 @@ function stats_update()
 
 
 
-# Lookup Duplicate Arc
+# Make Arc FileName
 #-----------------------------------------------------------------------------------------------------------------------------------
 
-function arc_dup_lookup()
+function arc_mkfn()
-{
-    dup_name_ptr=${1}
-     file_prefix=${2}
-     file_suffix=${3}
-
-
-    if [[ -f "${file_prefix}${file_suffix}" ]]
-    then
-	i=1
-
-	while [[ -f "${file_prefix}-${i}.{file_suffix}" ]]
-	do
-	  i=$((${i} + 1))
-	done
-
-	dup_name="-${i}"
-	dup_cnt=$((${dup_cnt} + 1))
-    else
-	dup_name=""
-    fi
-
-    eval ${dup_name_ptr}=${dup_name}
-}
-
-
-
-
-# Add Arc
-#-----------------------------------------------------------------------------------------------------------------------------------
-
-function arc_add()
 {
     file_name_ptr=${1}
               dir=${2}
@@ -381,25 +419,16 @@ function arc_add()
               crc=${7}
                fp=${8}
            status=${9}
+           dup_id=${10}
 
 
     file_prefix="${dir}/${id} - ${title} (${reg}) [${lang}] {${crc}} <${fp}> ${status}"
-    file_suffix=".zip"
 
-    if [[ -f "${file_prefix}${file_suffix}" ]]
+    if [[ ${dup_id} == "" ]]
     then
-	arc_dup_lookup "dup_name" "${file_prefix}" "${file_suffix}"
-
-	if [[ "${status}" == "OK" ]]
-        then
-	    print_verbose "moving ${file_prefix}${file_suffix} to ${file_prefix}${dup_name}${file_suffix}"
-
-	    mv "${file_prefix}${file_suffix}" "${file_prefix}${dup_name}${file_suffix}"
-
-	    dup_name=""
-	fi
-    else
 	dup_name=""
+    else
+	dup_name="-${dup_id}"
     fi
 
     eval ${file_name_ptr}=\"${file_prefix}${dup_name}${file_suffix}\"
@@ -409,51 +438,16 @@ function arc_add()
 
 
 
-# Remove Arc
-#-----------------------------------------------------------------------------------------------------------------------------------
-
-#function arc_remove()
-#{
-#    file=${1}
-#
-#
-#    prefix=${file/\> ??*.zip/>}
-#
-#    arc_list=$(mktemp)
-#    ls ${prefix}* > ${arc_list}
-#
-#    arc_mode="scan"
-#
-#    while read arc
-#    do
-#      case "${arc_mode}"
-#      in
-#          
-#      esac
-#    done < ${arc_list}
-#
-#    \rm -f ${arc_list}
-#}
-
-
-
-
-
 # Build Arc
 #-----------------------------------------------------------------------------------------------------------------------------------
 
 function arc_build()
 {
-         id=${1}
+    file_name=${1}
-      title=${2}
+          rom=${2}
-        reg=${3}
+         file=${3}
-       lang=${4}
+         mode=${4}
-        crc=${5}
+          idx=${5}
-         fp=${6}
-     status=${7}
-        rom=${8}
-       file=${9}
-       mode=${10}
 
 
     if [[ "${mode}" == "correct" ]]
@@ -463,15 +457,14 @@ function arc_build()
 	    print_verbose "removing ${file}"
 
 	    \rm -f  "${NDS_ROMS_IN}/${file}"
-
-#	    arc_remove "${file}"
 	fi
 
-	arc_add "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc}" "${fp}" "${status}"
-
 	print_verbose "zipping ${rom} into ${file_name}"
 
-	zip -m9 "${file_name}" "${rom}" >/dev/null 2>&1
+	zip -m9 "${file_name}" "${rom}"  >/dev/null 2>&1 &
+
+	job_pid[${job_id}]=$!
+	   dat_pid[${idx}]=$!
     else
 	print_verbose "removing ${rom}"
 
@@ -488,24 +481,16 @@ function arc_build()
 
 function arc_rename()
 {
-         id=${1}
+    file_name=${1}
-      title=${2}
+         file=${2}
-        reg=${3}
+       action=${3}
-       lang=${4}
+         mode=${4}
-        crc=${5}
-         fp=${6}
-     status=${7}
-       file=${8}
-     action=${9}
-       mode=${10}
 
 
     if [[ "${mode}" == "correct" ]]
     then
 	if [[ ${action} != "keep_status" ]]
 	then
-	    arc_add "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc}" "${fp}" "${status}"
-
 	    if [[ ${in_place} == "yes" ]]
             then
 		print_verbose "moving ${file} to ${file_name}"
@@ -524,6 +509,34 @@ function arc_rename()
 
 
 
+# Move Duplicate
+#-----------------------------------------------------------------------------------------------------------------------------------
+
+function dup_mv()
+{
+    file_name_trg=${1}
+    file_name_src=${2}
+             mode=${3}
+	      pid=${4}
+
+
+    if [[ "${mode}" == "correct" ]]
+    then
+	if [[ "${pid}" != "0" ]]
+        then
+	    wait "${pid}"
+	fi
+
+	print_verbose "moving ${file_name_src} to ${file_name_trg}"
+
+	mv "${file_name_src}" "${file_name_trg}"
+    fi
+}
+
+
+
+
+
 # Extract ROM
 #-----------------------------------------------------------------------------------------------------------------------------------
 
@@ -554,7 +567,7 @@ function rom_extract()
 	;;
 		
 	(*)
-	    echo "Internal error [$1] !";
+	    echo "rom_extract internal error [$1] !";
 	    exit 1
 	;;
     esac
@@ -564,11 +577,47 @@ function rom_extract()
 
 
 
+# Remove .tmp suffix
+#-----------------------------------------------------------------------------------------------------------------------------------
+
+function tmp_mv()
+{
+    echo "removing .tmp suffix..."
+
+       IFS=' '
+    mv_cnt=0
+   
+    for file_src in ${NDS_ROMS_OUT}/*.tmp
+    do
+      if [[ "${file_src}" != "${NDS_ROMS_OUT}/*.tmp" ]]
+      then
+	  file_trg=${file_src/.zip.tmp/.zip}
+
+	  print_verbose "move ${file_src} to ${file_trg}"
+
+	  mv "${file_src}" "${file_trg}"
+
+	  mv_cnt=$((${mv_cnt} + 1))
+
+	  printf "moved: %4d\r" "${mv_cnt}"
+      fi
+    done
+}
+
+
+
+
+
 # DSROM Mode
 #-----------------------------------------------------------------------------------------------------------------------------------
 
 function dsrom_mode()
 {
+    # Remove .tmp suffix
+    tmp_mv
+
+
+
     echo "scanning rom repository..."
     IFS=' '
 
@@ -590,7 +639,7 @@ function dsrom_mode()
        lang=${dat_lang[${idx}]}
         crc=${dat_crc[${idx}]}
 
-      regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..\.zip$"
+      regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..[-]*[0-9]*\.zip$"
 
 
       # Type of archive ?
@@ -619,20 +668,35 @@ function dsrom_mode()
 
 function test_correct_mode()
 {
-echo "scanning rom repository..."
+    if [[ ${in_place} == "yes" ]]
-IFS=' '
+    then
+	file_suffix=".zip.tmp"
+    else
+	file_suffix=".zip"
+    fi
 
-tmp_file=$(mktemp)
+    # Initialize Job Control
-idx_old=0
+    job_init
 
-ls ${NDS_ROMS_IN} > ${tmp_file}
+    # Remove .tmp suffix
+    tmp_mv
 
-while read file
+    echo "scanning rom repository..."
-do
+    IFS=' '
-  set ${file:0:4}
 
-   id=$1
+    tmp_file=$(mktemp)
-  idx=${id/*(0)/}
+    idx_old=0
+
+    ls ${NDS_ROMS_IN} > ${tmp_file}
+
+    while read file
+    do
+      job_switch "${job_id}"
+
+      set ${file:0:4}
+
+       id=$1
+      idx=${id/*(0)/}
 
   # Get info from dat
 
@@ -641,8 +705,7 @@ do
     lang=${dat_lang[${idx}]}
      crc=${dat_crc[${idx}]}
 
-  regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..\.zip$"
+  regex=".... - .* \(.*\) \[.*\] \{.*\} <.*> ..[-]*[0-9]*\.zip$"
-
 
  # Type of archive ?
 
@@ -752,12 +815,34 @@ do
 
 
 
-  # Update Dat Info
+  # Duplicate handling & Update Dat Info
 
-  if [[ "${dat_status[${idx}]}" != "OK" ]]
+  if [[ "${dat_status[${idx}]}" != "?" ]]
   then
-          dat_fp[${idx}]=${fp}
+      dat_dup[${idx}]=$((${dat_dup[${idx}]} + 1))
-      dat_status[${idx}]=${status}
+               dup_id="${dat_dup[${idx}]}"
+	      dup_cnt=$((${dup_cnt} + 1))
+
+      if [[ ( "${dat_status[${idx}]}" == "KO" ) && ( "${status}" == "OK" ) ]]
+      then
+	  dup_file_name="${dat_filename[${idx}]/${file_suffix}/-${dup_id}${file_suffix}}"
+
+	  dup_mv "${dup_file_name}" "${dat_filename[${idx}]}" "${mode}" "${dat_pid[${idx}]}"
+
+	  arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" ""
+
+              dat_fp[${idx}]=${fp}
+	  dat_status[${idx}]=${status}
+	dat_filename[${idx}]=${file_name}
+      fi
+
+      arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" "${dup_id}"
+  else
+      arc_mkfn "file_name" "${NDS_ROMS_OUT}" "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}" "${status}" ""
+
+            dat_fp[${idx}]=${fp}
+        dat_status[${idx}]=${status}
+      dat_filename[${idx}]=${file_name}
   fi
 
 
@@ -766,9 +851,9 @@ do
 
   if [[ ( "${arc_type}" != "rx3" ) || ( ${rebuild} == "yes" ) ]]
   then
-      arc_build  "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}"  "${status}" "${rom}"  "${file}"   "${mode}"
+      arc_build  "${file_name}" "${rom}"  "${file}"   "${mode}" "${idx}"
   else
-      arc_rename "${id}" "${title}" "${reg}" "${lang}" "${crc2}" "${fp}"  "${status}" "${file}" "${action}" "${mode}"
+      arc_rename "${file_name}" "${file}" "${action}" "${mode}"
   fi
 
 
@@ -794,10 +879,25 @@ done < ${tmp_file}
 
 wait
 
-  if [[ "${verbose}" != "yes" ]]
+if [[ "${verbose}" != "yes" ]]
-  then
+then
-      echo "                                                                                       "
+    echo "                                                                                       "
-  fi
+fi
+
+echo
+
+
+
+    # Remove .tmp suffix
+    #-----------------------------------------------------------------------------------------------------------------------------------
+
+    if [[ ${in_place} == "yes" ]]
+    then
+	tmp_mv
+    fi
+
+    # Initialize Job Control
+    job_deinit
 
 
 
@@ -906,7 +1006,14 @@ do
 
   if [[ "${idx}" != "xxxx" ]]
   then
-      echo -ne "$2\r"
+      echo -ne "$2"
+
+      if [[ "${verbose}" == "yes" ]]
+      then
+	  echo
+      else
+	  printf "\r"
+      fi
 
              dat_rn[${idx}]="${1}"
              dat_id[${idx}]="${2}"
@@ -915,8 +1022,12 @@ do
            dat_lang[${idx}]="${5}"
       dat_save_type[${idx}]="${6}"
             dat_crc[${idx}]="${7}"
+
              dat_fp[${idx}]="????"
          dat_status[${idx}]="?"
+            dat_dup[${idx}]="0"
+       dat_filename[${idx}]=""
+            dat_pid[${idx}]="0"
 
       loaded_cnt=$((${loaded_cnt} + 1))
   fi
@@ -925,6 +1036,7 @@ done < ${NDS_DAT}
 
 
 #-----------------------------------------------------------------------------------------------------------------------------------
+
 if [[ "${mode}" == "dsrom" ]]
 then
     dsrom_mode