- Total loader rewrite using byte data type: Raw file loading implemenred,
- Add MQ character set support, - Add Character Set option (-cs).
This commit is contained in:
agibert
parent
542f5b9bd6
commit
ffb13ee948
316
MQSLoad.java
316
MQSLoad.java
@ -1,7 +1,7 @@
|
||||
// $RCSfile: MQSLoad.java,v $
|
||||
// $Revision: 1.15 $
|
||||
// $Revision: 1.16 $
|
||||
// $Name: $
|
||||
// $Date: 2008/12/04 15:02:16 $
|
||||
// $Date: 2008/12/09 15:01:02 $
|
||||
// $Author: agibert $
|
||||
|
||||
/*
|
||||
@ -35,15 +35,16 @@ import com.ibm.mq.*;
|
||||
|
||||
public class MQSLoad
|
||||
{
|
||||
private String MQSLoad_Revision = "$Revision: 1.15 $";
|
||||
private String MQSLoad_Revision = "$Revision: 1.16 $";
|
||||
private String MQSLoad_Tag = "$Name: $";
|
||||
private String MQSLoad_Date = "$Date: 2008/12/04 15:02:16 $";
|
||||
private String MQSLoad_Date = "$Date: 2008/12/09 15:01:02 $";
|
||||
private String MQSLoad_Author = "$Author: agibert $";
|
||||
private MQQueueManager QMng = null;
|
||||
private String QMng_Name = "";
|
||||
private MQQueue MsgQ;
|
||||
private String MsgQ_Name;
|
||||
private int MsgQ_Open_Options = MQC.MQOO_OUTPUT | MQC.MQOO_FAIL_IF_QUIESCING;
|
||||
private int Character_Set = 1208;
|
||||
private int Sleep_Time = 0;
|
||||
private String Field_Break = "";
|
||||
private String Message_Break = "\r\n";
|
||||
@ -54,6 +55,8 @@ public class MQSLoad
|
||||
private boolean Keep_Message_Break = false;
|
||||
private String File_Name;
|
||||
private int Msg_Counter = 0;
|
||||
private int Msg_Read_Nb = 0;
|
||||
private int Msg_Skiped_Nb = 0;
|
||||
|
||||
|
||||
|
||||
@ -220,6 +223,30 @@ public class MQSLoad
|
||||
throw new Exception();
|
||||
}
|
||||
}
|
||||
else if ( args[argc].equals( "-cs"))
|
||||
{
|
||||
if( argc < ( args.length + 1))
|
||||
{
|
||||
Character_Set = Integer.parseInt( args[++argc]);
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println( "Invalid number of command line options...");
|
||||
throw new Exception();
|
||||
}
|
||||
}
|
||||
else if ( args[argc].equals( "-st"))
|
||||
{
|
||||
if( argc < ( args.length + 1))
|
||||
{
|
||||
Sleep_Time = Integer.parseInt( args[++argc]);
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println( "Invalid number of command line options...");
|
||||
throw new Exception();
|
||||
}
|
||||
}
|
||||
else if( args[argc].equals( "-fb"))
|
||||
{
|
||||
if( argc < ( args.length + 1))
|
||||
@ -256,18 +283,6 @@ public class MQSLoad
|
||||
throw new Exception();
|
||||
}
|
||||
}
|
||||
else if ( args[argc].equals( "-st"))
|
||||
{
|
||||
if( argc < ( args.length + 1))
|
||||
{
|
||||
Sleep_Time = Integer.parseInt( args[++argc]);
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println( "Invalid number of command line options...");
|
||||
throw new Exception();
|
||||
}
|
||||
}
|
||||
else if ( args[argc].equals( "-ms"))
|
||||
{
|
||||
if( argc < ( args.length + 1))
|
||||
@ -428,6 +443,91 @@ public class MQSLoad
|
||||
|
||||
|
||||
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
//
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
private int Buffer_Lookup( byte[] Buffer, int Offset, int End, byte[] Message_Break)
|
||||
{
|
||||
int i = Offset;
|
||||
int j = 0;
|
||||
|
||||
|
||||
while( ( i < End) && ( j < Message_Break.length))
|
||||
{
|
||||
if( Buffer[i++] == Message_Break[j])
|
||||
{
|
||||
j++;
|
||||
}
|
||||
else
|
||||
{
|
||||
j = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if( j == Message_Break.length)
|
||||
{
|
||||
return( i - j);
|
||||
}
|
||||
else
|
||||
{
|
||||
return( -1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
//
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
private void Msg_Write( MQMessage Msg, byte[] Buffer, int Offset, int End, byte[] Field_Break) throws Exception
|
||||
{
|
||||
int cur_pos = Offset;
|
||||
int fld_break;
|
||||
|
||||
|
||||
/* scan into the buffer */
|
||||
while( cur_pos < End)
|
||||
{
|
||||
if( Field_Break.length == 0)
|
||||
{
|
||||
/* No Field Break */
|
||||
// System.out.println( "* Skip FB Lookup...");
|
||||
|
||||
fld_break = End;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Field Break Lookup */
|
||||
// System.out.println( "* FB Lookup: (" + fld_break + ")");
|
||||
|
||||
fld_break = Buffer_Lookup( Buffer, cur_pos, End, Field_Break);
|
||||
}
|
||||
|
||||
if( fld_break == -1)
|
||||
{
|
||||
/* Write the rest of the buffer */
|
||||
// System.out.println( "* Last Field");
|
||||
|
||||
fld_break = End;
|
||||
}
|
||||
|
||||
/* Write to MQ-Message */
|
||||
// System.out.println( "* Write Field: cur_pos(" + cur_pos + ") Len: (" + ( fld_break - cur_pos) + ")");
|
||||
|
||||
Msg.write( Buffer, cur_pos, ( fld_break - cur_pos));
|
||||
|
||||
cur_pos = fld_break + Field_Break.length;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
//
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
@ -483,17 +583,19 @@ public class MQSLoad
|
||||
//
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
private int Put_Msg( MQMessage Output_Msg, int Msg_Id, int Msg_Nb, int Msg_Skip, int Msg_Count) throws Exception
|
||||
private int Msg_Put( MQMessage Output_Msg, int Msg_Id, int Msg_Nb, int Msg_Skip, int Msg_Count) throws Exception
|
||||
{
|
||||
if( Msg_Id < Msg_Skip)
|
||||
{
|
||||
Counter_Print( "*");
|
||||
Msg_Skiped_Nb++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if( ( Msg_Count != 0) && ( Msg_Id >= ( Msg_Skip + Msg_Count)))
|
||||
{
|
||||
Counter_Print( "#");
|
||||
Msg_Skiped_Nb++;
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -504,13 +606,24 @@ public class MQSLoad
|
||||
Counter_Print( ".");
|
||||
|
||||
Msg_Nb++;
|
||||
Msg_Read_Nb++;
|
||||
|
||||
Stand_By( Sleep_Time);
|
||||
}
|
||||
}
|
||||
|
||||
Msg_Id++;
|
||||
|
||||
if( ( Msg_Count != 0) && ( Msg_Id >= ( Msg_Skip + Msg_Count)))
|
||||
{
|
||||
/* Last Message to be read */
|
||||
return( -1);
|
||||
}
|
||||
else
|
||||
{
|
||||
return( Msg_Nb);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -520,133 +633,131 @@ public class MQSLoad
|
||||
//
|
||||
//-------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
private int Load_File( RandomAccessFile Input_File, int Msg_Nb, int Msg_Skip, int Msg_Count) throws Exception
|
||||
private void File_Load( RandomAccessFile Input_File, int Msg_Skip, int Msg_Count) throws Exception
|
||||
{
|
||||
byte input_char;
|
||||
MQMessage output_msg = new MQMessage();
|
||||
int buffer_size, cur_buffer_size, window_size, cur_window_size, prefetch_size, buffer_offset, read_size;
|
||||
int cur_pos, end_pos, next_message, next_field;
|
||||
byte[] buffer;
|
||||
byte[] message_break = Message_Break.getBytes();
|
||||
byte[] field_break = Field_Break.getBytes();
|
||||
byte[] message_tail = Message_Tail.getBytes();
|
||||
int buf_size, work_win_size, break_win_size, prefetch_size, fetch_size;
|
||||
int cur_buf_size, cur_work_win_size, cur_pos, msg_break;
|
||||
int msg_id = 0;
|
||||
byte[] buffer_byte;
|
||||
String buffer_string, out_string = null;
|
||||
int msg_nb = 0;
|
||||
|
||||
|
||||
try
|
||||
{
|
||||
output_msg.format = MQC.MQFMT_STRING;
|
||||
output_msg.characterSet = Character_Set;
|
||||
|
||||
work_win_size = 1024 * 64;
|
||||
break_win_size = Field_Break.length() + message_break.length;
|
||||
|
||||
/* Default window size */
|
||||
window_size = 1024 * 64;
|
||||
prefetch_size = Field_Break.length() + Message_Break.length();
|
||||
buffer_size = window_size + prefetch_size;
|
||||
buf_size = work_win_size + break_win_size;
|
||||
buffer = new byte[buf_size];
|
||||
|
||||
buffer_offset = 0;
|
||||
|
||||
buffer_byte = new byte[buffer_size];
|
||||
prefetch_size = 0;
|
||||
|
||||
do
|
||||
{
|
||||
read_size = Input_File.read( buffer_byte, buffer_offset, buffer_size - buffer_offset);
|
||||
// System.out.println( "Load buffer read_size: (" + read_size + ") asked: (" + (buffer_size - buffer_offset) + ")");
|
||||
|
||||
if( read_size == -1)
|
||||
/* Fill the buffer */
|
||||
do
|
||||
{
|
||||
read_size = 0;
|
||||
fetch_size = Input_File.read( buffer, prefetch_size, buf_size - prefetch_size);
|
||||
// System.out.println( "Load buffer fetch_size: (" + fetch_size + ") asked: (" + (buf_size - prefetch_size) + ")");
|
||||
|
||||
if( fetch_size == -1)
|
||||
{
|
||||
fetch_size = 0;
|
||||
}
|
||||
|
||||
/* Compute current buffer and window size */
|
||||
cur_window_size = Math.min( ( read_size + buffer_offset), window_size);
|
||||
cur_buffer_size = ( read_size + buffer_offset);
|
||||
/* compute the current buffer size, work window size and new prefetch size */
|
||||
cur_buf_size = prefetch_size + fetch_size;
|
||||
cur_work_win_size = Math.min( work_win_size, cur_buf_size);
|
||||
|
||||
if( cur_buf_size < break_win_size)
|
||||
{
|
||||
prefetch_size = cur_buf_size;
|
||||
}
|
||||
|
||||
buffer_string = new String( buffer_byte, 0, cur_buffer_size);
|
||||
cur_pos = 0;
|
||||
|
||||
// System.out.println( " CurWS: (" + cur_window_size + ") CurBS: (" + cur_buffer_size + ") Buffer: [" + buffer_string + "]");
|
||||
// System.out.println( " Cur Work Win Size: (" + cur_work_win_size + ") Cur Buf Size: (" + cur_buf_size + ") Prefetch Size: (" + prefetch_size + ")");
|
||||
}
|
||||
while( ( fetch_size != 0) && ( cur_buf_size < break_win_size));
|
||||
|
||||
while( cur_pos < cur_window_size)
|
||||
|
||||
|
||||
/* scan into the work window */
|
||||
while( cur_pos < cur_work_win_size)
|
||||
{
|
||||
// System.out.println( "- CurPos: (" + cur_pos + ")");
|
||||
next_message = buffer_string.indexOf( Message_Break, cur_pos);
|
||||
|
||||
if( ( next_message == -1) || ( next_message > cur_window_size))
|
||||
msg_break = Buffer_Lookup( buffer, cur_pos, cur_buf_size, message_break);
|
||||
|
||||
if( msg_break == -1)
|
||||
{
|
||||
next_message = cur_window_size + 1;
|
||||
end_pos = cur_window_size;
|
||||
/* write the message to the MQ buffer */
|
||||
// System.out.println( "+ No msg break");
|
||||
// System.out.println( "* Write MQ Buffer Offset: (" + cur_pos + ") End: (" + cur_work_win_size + ")");
|
||||
Msg_Write( output_msg, buffer, cur_pos, cur_work_win_size, field_break);
|
||||
|
||||
cur_pos = cur_work_win_size;
|
||||
}
|
||||
else
|
||||
{
|
||||
// System.out.println( "+ Get message next_message: (" + next_message + ")");
|
||||
end_pos = next_message;
|
||||
}
|
||||
// System.out.println( "+ Get msg break : (" + msg_break + ")");
|
||||
|
||||
while( cur_pos < end_pos)
|
||||
{
|
||||
// System.out.println( "+ CurPos: (" + cur_pos + ") end_pos: (" + end_pos + ")");
|
||||
if( Field_Break.length() > 0)
|
||||
{
|
||||
next_field = buffer_string.indexOf( Field_Break, cur_pos);
|
||||
}
|
||||
else
|
||||
{
|
||||
next_field = -1;
|
||||
}
|
||||
/* write the message to the MQ buffer */
|
||||
Msg_Write( output_msg, buffer, cur_pos, msg_break, field_break);
|
||||
|
||||
if( ( next_field == -1) || ( next_field > end_pos))
|
||||
{
|
||||
out_string = buffer_string.substring( cur_pos, end_pos);
|
||||
cur_pos = end_pos;
|
||||
}
|
||||
else
|
||||
{
|
||||
// System.out.println( "+ Get field next_field: (" + next_field + ")");
|
||||
out_string = buffer_string.substring( cur_pos, next_field);
|
||||
cur_pos = next_field + Field_Break.length();
|
||||
}
|
||||
|
||||
output_msg.writeString( out_string);
|
||||
// System.out.println( "* Msg: [" + out_string + "]");
|
||||
|
||||
}
|
||||
|
||||
if( next_message <= cur_window_size)
|
||||
{
|
||||
/* A message break has been found: send the data... */
|
||||
// System.out.println( "* Write Msg CurPos: (" + cur_pos + ")");
|
||||
output_msg.writeString( Message_Tail);
|
||||
/* write the message tail to the MQ buffer */
|
||||
output_msg.write( message_tail, 0, message_tail.length);
|
||||
|
||||
if( Keep_Message_Break)
|
||||
{
|
||||
output_msg.writeString( Message_Break);
|
||||
/* write the message break to the MQ buffer */
|
||||
// System.out.println( "* Write MQ Buffer Offset: (" + cur_pos + ") End: (" + ( msg_break + message_break.length) + ")");
|
||||
output_msg.write( buffer, msg_break, message_break.length);
|
||||
}
|
||||
|
||||
// System.out.println( "(" + Msg_Nb + ")");
|
||||
Msg_Nb = Put_Msg( output_msg, msg_id, Msg_Nb, Msg_Skip, Msg_Count);
|
||||
/* send the message */
|
||||
// System.out.println( "* Put Msg()");
|
||||
|
||||
if( ( msg_nb = Msg_Put( output_msg, msg_id, msg_nb, Msg_Skip, Msg_Count)) == -1)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
msg_id++;
|
||||
out_string = null;
|
||||
cur_pos += Message_Break.length();
|
||||
cur_pos = msg_break + message_break.length;
|
||||
}
|
||||
}
|
||||
|
||||
buffer_offset = cur_buffer_size - cur_pos;
|
||||
// System.out.println( "% CurPos: (" + cur_pos + ") BufOff: (" + buffer_offset + ")");
|
||||
|
||||
if( buffer_offset > 0)
|
||||
|
||||
prefetch_size = cur_buf_size - cur_pos;
|
||||
// System.out.println( "- Prefetch size: (" + prefetch_size + ")");
|
||||
|
||||
if( prefetch_size > 0)
|
||||
{
|
||||
/* Copy back prefetch area */
|
||||
System.arraycopy( buffer_byte, cur_pos, buffer_byte, 0, buffer_offset);
|
||||
System.arraycopy( buffer, cur_pos, buffer, 0, prefetch_size);
|
||||
}
|
||||
}
|
||||
while( cur_buffer_size > 0);
|
||||
while( cur_buf_size > 0);
|
||||
|
||||
if( out_string != null)
|
||||
|
||||
|
||||
if( output_msg.getMessageLength() > 0)
|
||||
{
|
||||
// System.out.println( "* Flushing Msg");
|
||||
Msg_Nb = Put_Msg( output_msg, msg_id, Msg_Nb, Msg_Skip, Msg_Count);
|
||||
/* flush the message */
|
||||
// System.out.println( "* Put Msg()");
|
||||
Msg_Put( output_msg, msg_id, msg_nb, Msg_Skip, Msg_Count);
|
||||
}
|
||||
|
||||
return( Msg_Nb);
|
||||
return;
|
||||
}
|
||||
|
||||
catch( Exception Expt)
|
||||
@ -665,8 +776,8 @@ public class MQSLoad
|
||||
|
||||
private void Print_Usage( ) throws Exception
|
||||
{
|
||||
System.out.println( "Usage: MQSLoad [-qm \"Output_QueueMng_Name\"] [-st \"sleep_time\"] [-fb \"field_break\"] [-mb \"message_break\"] [-mt \"message_tail\"] [-kmb] [-ms \"message_skip\"] [-mc \"message_count\"] [-rc \"repeat_count\"] <Output_MsgQueue_Name> <Input_File_Name>");
|
||||
System.out.println( " Default: Output QueueMng Name: (" + QMng_Name + ") Sleep Time: (" + Sleep_Time + ") Field Break: (" + Str_UnFormat( Field_Break) + ") Message Break: (" + Str_UnFormat( Message_Break) + ") Message Tail: (" + Str_UnFormat( Message_Tail) + ") Keep Message Break: (" + Keep_Message_Break + ") Message Skip: (" + Message_Skip + ") Message Count: (" + Message_Count + ") Repeat Count: (" + Repeat_Count + ")");
|
||||
System.out.println( "Usage: MQSLoad [-qm \"Output_QueueMng_Name\"] [-cs \"character_set\"] [-st \"sleep_time\"] [-fb \"field_break\"] [-mb \"message_break\"] [-mt \"message_tail\"] [-kmb] [-ms \"message_skip\"] [-mc \"message_count\"] [-rc \"repeat_count\"] <Output_MsgQueue_Name> <Input_File_Name>");
|
||||
System.out.println( " Default: Output QueueMng Name: (" + QMng_Name + ") Character Set: (" + Character_Set + ") Sleep Time: (" + Sleep_Time + ") Field Break: (" + Str_UnFormat( Field_Break) + ") Message Break: (" + Str_UnFormat( Message_Break) + ") Message Tail: (" + Str_UnFormat( Message_Tail) + ") Keep Message Break: (" + Keep_Message_Break + ") Message Skip: (" + Message_Skip + ") Message Count: (" + Message_Count + ") Repeat Count: (" + Repeat_Count + ")");
|
||||
}
|
||||
|
||||
|
||||
@ -680,7 +791,7 @@ public class MQSLoad
|
||||
private void Print_Args( ) throws Exception
|
||||
{
|
||||
System.out.println( "Output QueueMng Name: (" + QMng_Name + ") Output MsgQueue Name: (" + MsgQ_Name + ") Input File Name: (" + File_Name + ")");
|
||||
System.out.println( "Sleep Time: (" + Sleep_Time + ") Field Break: (" + Str_UnFormat( Field_Break) + ") Message Break: (" + Str_UnFormat( Message_Break) + ") Message Tail: (" + Str_UnFormat( Message_Tail) + ") Keep Message Break: (" + Keep_Message_Break + ") Message Skip: (" + Message_Skip + ") Message Count: (" + Message_Count + ") Repeat Count: (" + Repeat_Count + ")");
|
||||
System.out.println( "Character Set: (" + Character_Set + ") Sleep Time: (" + Sleep_Time + ") Field Break: (" + Str_UnFormat( Field_Break) + ") Message Break: (" + Str_UnFormat( Message_Break) + ") Message Tail: (" + Str_UnFormat( Message_Tail) + ") Keep Message Break: (" + Keep_Message_Break + ") Message Skip: (" + Message_Skip + ") Message Count: (" + Message_Count + ") Repeat Count: (" + Repeat_Count + ")");
|
||||
}
|
||||
|
||||
|
||||
@ -693,7 +804,6 @@ public class MQSLoad
|
||||
|
||||
public MQSLoad( String args[])
|
||||
{
|
||||
int msg_nb = 0;
|
||||
int loop;
|
||||
long time_begin, time_end;
|
||||
double time_elapsed, speed;
|
||||
@ -714,7 +824,6 @@ public class MQSLoad
|
||||
System.out.println( "MQS Load Starting...");
|
||||
|
||||
Arg_Parse( args);
|
||||
|
||||
Print_Args();
|
||||
|
||||
MQSInit();
|
||||
@ -745,19 +854,20 @@ public class MQSLoad
|
||||
}
|
||||
}
|
||||
|
||||
msg_nb = Load_File( input_file, msg_nb, Message_Skip, Message_Count);
|
||||
File_Load( input_file, Message_Skip, Message_Count);
|
||||
}
|
||||
|
||||
time_end = System.currentTimeMillis();
|
||||
|
||||
time_elapsed = ( time_end - time_begin) / 1000.0;
|
||||
speed = msg_nb / time_elapsed;
|
||||
speed = Msg_Read_Nb / time_elapsed;
|
||||
|
||||
nf.setMinimumFractionDigits(2);
|
||||
nf.setMaximumFractionDigits(2);
|
||||
|
||||
System.out.println( "\n");
|
||||
System.out.println( "Loaded Message Nb: (" + msg_nb
|
||||
System.out.println( "Loaded Message Nb: (" + Msg_Read_Nb
|
||||
+ ") Skiped Messages Nb: (" + Msg_Skiped_Nb
|
||||
+ ") Elapsed Time: (" + nf.format(time_elapsed)
|
||||
+ ") s Speed: (" + nf.format(speed)
|
||||
+ ") msg/s");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user