modules/ud/ud_process_stream.c
/* [<][>][^][v][top][bottom][index][help] */
FUNCTIONS
This source file includes following functions.
- ud_parse_init
- ud_parse_free
- line_continuation
- ud_replace_substring
- ud_split_attribute
- each_attribute_print
- print_object
- escape_apostrophes
- line_type
- UD_parse_object
- each_attribute_2_pass
- ud_normalize
- report_transaction
- process_nrtm
- process_updates
- process_transaction
- UD_process_stream
1 /***************************************
2 $Revision: 1.50 $
3
4 Functions to process data stream( file, network socket, etc.)
5
6 Status: NOT REVUED, NOT TESTED
7
8 Author(s): Chris Ottrey, Andrei Robachevsky
9
10 ******************/ /******************
11 Modification History:
12 andrei (17/01/2000) Created.
13 ******************/ /******************
14 Copyright (c) 2000 RIPE NCC
15
16 All Rights Reserved
17
18 Permission to use, copy, modify, and distribute this software and its
19 documentation for any purpose and without fee is hereby granted,
20 provided that the above copyright notice appear in all copies and that
21 both that copyright notice and this permission notice appear in
22 supporting documentation, and that the name of the author not be
23 used in advertising or publicity pertaining to distribution of the
24 software without specific, written prior permission.
25
26 THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
27 ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
28 AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
29 DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
30 AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
31 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 ***************************************/
33 #include <sys/types.h>
34 #include <sys/socket.h>
35 #include <netdb.h>
36 #include <arpa/inet.h>
37 #include <unistd.h>
38 #include <sys/stat.h>
39 #include <fcntl.h>
40 #include <string.h>
41 #include "constants.h"
42 #include "query_command.h"
43 #include "ud.h"
44 #include "ud_int.h"
45 #include "ud_tr.h"
46 #include "timediff.h"
47
48 typedef enum _Line_Type_t {
49 LINE_ATTRIBUTE,
50 LINE_COMMENT,
51 LINE_EMPTY,
52 LINE_EOF,
53 LINE_ADD,
54 LINE_UPD,
55 LINE_DEL,
56 LINE_OVERRIDE_ADD,
57 LINE_OVERRIDE_UPD,
58 LINE_OVERRIDE_DEL,
59 LINE_ACK
60 } Line_Type_t;
61
62 /* Maximum number of objects(serials) we can consume at a time */
63 #define SBUNCH 1000
64
65 static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason);
66 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
67 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
68 static int process_transaction(UD_stream_t *ud_stream, Obj_parse_t *parse, int operation, long transaction_id);
69
70 static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr);
71 static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse);
72 static void ud_parse_free(Obj_parse_t *parse);
73 static int line_continuation(char *line);
74 static GString *escape_apostrophes(GString *text);
75 static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff);
76 static int ud_replace_substring(GString *gstring, char *old, char *new);
77 static int ud_normalize(Obj_parse_t *parse);
78
79 /* Delimiters that separate list members, both RPS(,) and legacy( ) */
80 #define ATTR_DELIMITERS " ,"
81
82
83 static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse){
/* [<][>][^][v][top][bottom][index][help] */
84 bzero(parse, sizeof(Obj_parse_t));
85 parse->garbage = 0;
86 parse->sql_connection = sql_connection;
87 }
88
89 static void ud_parse_free(Obj_parse_t *parse){
/* [<][>][^][v][top][bottom][index][help] */
90 UT_free(parse->nic);
91 UT_free(parse->object_name);
92 }
93
94
95 /******************************************************************
96 * int line_continuation() *
97 * *
98 * Checks if the line starts with one of line continuation signs *
99 * Returns 1 if this is line continuation, 0 otherwise *
100 * *
101 * ****************************************************************/
102 static int line_continuation(char *line)
/* [<][>][^][v][top][bottom][index][help] */
103 {
104 switch(*line) {
105 case ' ':
106 case '\t':
107 case '+':
108 return(1); /* these indicate line continuation */
109 default: return(0);
110 }
111
112 }
113
114 /******************************************************************
115 * int ud_replace_substring() *
116 * *
117 * replace a subsring in a string *
118 * Returns -1 if no match was found *
119 * *
120 * ****************************************************************/
121 static int ud_replace_substring(GString *gstring, char *old, char *new)
/* [<][>][^][v][top][bottom][index][help] */
122 {
123 int i, i0=-1, j=0;
124
125 for (i=0; ((i < gstring->len) && (old[j] != '\0')); i++){
126 if(gstring->str[i]==old[j]){
127 if(i0==-1)i0=i;
128 j++;
129 } else {
130 i0=-1;
131 j=0;
132 }
133 }
134 /* if we haven't reached the end of the old substring */
135 /* or no match occured return -1 */
136 if((old[j] != '\0') || (i0==-1)) return (-1);
137 g_string_erase(gstring, i0, j);
138 g_string_insert(gstring, i0, new);
139 return(0);
140 }
141
142 /************************************************************
143 * *
144 * The function to splits attribute value into multiple *
145 * words and appends them as attr_type - attr_valu pairs *
146 * to the attr_list *
147 * *
148 * *
149 ************************************************************/
150 static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr)
/* [<][>][^][v][top][bottom][index][help] */
151 {
152 char *token;
153 char *split;
154 char *value, *n;
155 Attribute_t *attr_split;
156 GSList *the_list = attr_list;
157
158
159 /* check for line continuation (+) */
160 if (strncmp(attr->value, "+", 1) == 0) attr->value++;
161 /* check for end-of-line comments */
162 n = index(attr->value, '#');
163 /* if there is no comment check for trailing \n */
164 if(n == NULL) n = index(attr->value, '\n');
165 /* now copy the clean value into the attribute */
166 if(n == NULL) value = g_strdup(attr->value);
167 else value = g_strndup(attr->value, (n - attr->value));
168
169 token=value;
170 while((split=strsep(&token, ATTR_DELIMITERS))){
171 attr_split = attribute_new1(attr->type, split);
172 if (attr_split) the_list = g_slist_append(the_list, attr_split);
173 }
174 free(value);
175 attribute_free(attr, NULL);
176
177 return(the_list);
178 }
179
180 /* XXX */
181 static void each_attribute_print(void *element_data, void *tr_ptr)
/* [<][>][^][v][top][bottom][index][help] */
182 {
183
184 Attribute_t *attr = (Attribute_t *)element_data;
185
186 fprintf(stderr, "[%d|%s]\n", attr->type, attr->value);
187
188 }
189
190 /* XXX */
191 static void print_object(Object_t *obj)
/* [<][>][^][v][top][bottom][index][help] */
192 {
193 g_slist_foreach(obj->attributes, each_attribute_print, NULL);
194 fprintf(stderr, ">>>>>\n%s\n", obj->object->str);
195 }
196
197
198 /******************************************************************
199 * GString *escape_apostrophes() *
200 * Escapes apostrophes in the text so they do not confuse printf *
201 * functions and don't corrupt SQL queries *
202 * *
203 * *****************************************************************/
204 static GString *escape_apostrophes(GString *text) {
/* [<][>][^][v][top][bottom][index][help] */
205 int i;
206 for (i=0; i < text->len; i++) {
207 if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
208 text = g_string_insert_c(text, i, '\\');
209 i++;
210 }
211 }
212 return(text);
213 } /* escape_apostrophes() */
214
215
216 /******************************************************************
217 * Line_Type_t line_type(e) *
218 * Determines the line type analysing the first letters *
219 * *
220 * ****************************************************************/
221 static Line_Type_t line_type(const char *line, long *transaction_id) {
/* [<][>][^][v][top][bottom][index][help] */
222
223 if (strncmp(line, "# EOF", 4) == 0) return(LINE_EOF);
224 if (strncmp(line, "#", 1) == 0) return(LINE_COMMENT);
225 if (strcmp(line, "\n") == 0) return(LINE_EMPTY);
226
227 if (strncmp(line, "ACK", 3) == 0) {
228 *transaction_id = atol(line+3);
229 return(LINE_ACK);
230 }
231 if (strncmp(line, "ADD_OVERRIDE", 12) == 0) {
232 *transaction_id = atol(line+12);
233 return(LINE_OVERRIDE_ADD);
234 }
235 if (strncmp(line, "UPD_OVERRIDE", 12) == 0) {
236 *transaction_id = atol(line+12);
237 return(LINE_OVERRIDE_UPD);
238 }
239 if (strncmp(line, "DEL_OVERRIDE", 12) == 0) {
240 *transaction_id = atol(line+12);
241 return(LINE_OVERRIDE_DEL);
242 }
243
244 if (strncmp(line, "ADD", 3) == 0) {
245 *transaction_id = atol(line+3);
246 return(LINE_ADD);
247 }
248 if (strncmp(line, "UPD", 3) == 0) {
249 *transaction_id = atol(line+3);
250 return(LINE_UPD);
251 }
252 if (strncmp(line, "DEL", 3) == 0) {
253 *transaction_id = atol(line+3);
254 return(LINE_DEL);
255 }
256
257 /* Otherwise this is an attribute */
258 return(LINE_ATTRIBUTE);
259
260 } /* line_type() */
261
262 /******************************************************************
263 * Object_t *UD_parse_object() *
264 * *
265 * Parses the object accepting line by line *
266 * Stores the object and list of attributes in Obj_parse_t struct *
267 * *
268 * ****************************************************************/
269 static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff)
/* [<][>][^][v][top][bottom][index][help] */
270 {
271 GString *g_line_buff;
272 Attribute_t *attr;
273
274 /* in case of garbage */
275 if(parse->garbage) return(NULL);
276
277 if(parse->obj==NULL) {
278 /* which means this is the start - create a new object*/
279 parse->obj = object_new(line_buff);
280 /* if this is not a class attribute - this is a garbage till the next blank line */
281 if(parse->obj==NULL) {
282 parse->garbage = 1;
283 return(NULL);
284 }
285 }
286
287 /* allocate buffer or the string */
288 if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){
289 ER_perror(FAC_UD, UD_MEM, "cannot allocate gstring");
290 die;
291 }
292 /* copy the input */
293 g_string_sprintf(g_line_buff, "%s", line_buff);
294 /* escape apostrophes in the input line */
295 g_line_buff=escape_apostrophes(g_line_buff);
296
297 attr = attribute_new(g_line_buff->str);
298 if (attr){
299 /* save this attribute for future line continuations */
300 /* skip attributes that are not stored in SQL database */
301 if(DF_get_update_query_type(attr->type)==UD_NULL_){
302 parse->current_attr = NULL;
303 attribute_free(attr, NULL);
304 }
305 else {
306 parse->current_attr = attr;
307 if((attr->type == A_MB) || (attr->type == A_NH)) {
308 /* insert it at the beginning */
309 /* mnt-by should go before member-of to allow correct membership autorization (still done in RIPupd) */
310 /* nic-hdl should go before any admin-c, tech-c to prevent errrors in self referencing role objects */
311 parse->obj->attributes = g_slist_insert(parse->obj->attributes, parse->current_attr, 1);
312 } else {
313 /* append it to the attribute list */
314 parse->obj->attributes = g_slist_append(parse->obj->attributes, parse->current_attr);
315 }
316 }
317 }
318 /* if this is not a valid attribute, then this may be unknown attr or line continuation */
319 else
320 if(line_continuation(g_line_buff->str))
321 {
322 /* if parse->attr == NULL, then this attr is not stored in SQL database - skip it */
323 if(parse->current_attr){
324 /* this is continuation of a previous attribute */
325 char *line_value, *new_value;
326 char *n;
327 /* normalize and append to current_attr->value */
328 /* check for end-of-line comments */
329 n = index(g_line_buff->str, '#');
330 /* if there is no comment check for trailing \n */
331 if(n == NULL) n = index(g_line_buff->str, '\n');
332 /* now copy the clean value into the attribute */
333 if(n == NULL) line_value = g_strdup(g_line_buff->str);
334 else line_value = g_strndup(g_line_buff->str, (n - g_line_buff->str));
335 /* replace line continuation character with ' ' */
336 *line_value = ' ';
337 new_value = g_strconcat(parse->current_attr->value, line_value, NULL);
338 parse->current_attr = attribute_upd(parse->current_attr, parse->current_attr->type, new_value);
339 UT_free(new_value);
340 UT_free(line_value);
341 }
342 }
343 /* otherwise, this is unknown attribute, so we need to break the continuation */
344 else parse->current_attr=NULL;
345 /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */
346 g_string_sprintfa(parse->obj->object, "%s", g_line_buff->str);
347 g_string_free(g_line_buff, TRUE);
348 return(parse->obj);
349 }
350
351 /******************************************************************
352 * int each_attribute_2_pass() *
353 * *
354 * Assigns nic-handles in case of AUTO *
355 * Splits attributes *
356 * * *
357 ******************************************************************/
358 void each_attribute_2_pass(void *element_data, void *ptr)
/* [<][>][^][v][top][bottom][index][help] */
359 {
360 Attribute_t *attr = (Attribute_t *)element_data;
361 Obj_parse_t *parse = (Obj_parse_t *)ptr;
362
363
364 /* Strip the white space */
365 g_strstrip(attr->value);
366
367 switch(attr->type){
368 case A_NH: /* nic-hdl */
369 /* Parse the string into nh structure */
370 /* In case of an AUTO NIC handle check the ID in the database */
371 /* Possible errors leave to core processing */
372 if(NH_parse(attr->value, &parse->nh_ptr) == 0) {
373 /* this is an AUTO nic handle */
374 /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsing nic handle: [%s]", UD_TAG, attr->value);*/
375 /* Check if we can allocate it */
376 if(NH_check(parse->nh_ptr, parse->sql_connection)>0){
377 /* Convert nh to the database format */
378 parse->nic = NH_convert(parse->nh_ptr);
379 /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsed and converted nic handle: [%s]", UD_TAG, nic); */
380 /* replace the auto nic with a real one */
381 if(ud_replace_substring(parse->obj->object, attr->value, parse->nic)==-1) {
382 ER_perror(FAC_UD, UD_BUG, "cannot replace auto nic handle");
383 die;
384 }
385 /* Update the attribute */
386 attr = attribute_upd(attr, attr->type, parse->nic);
387
388 }
389 }
390 parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
391 break;
392 case A_PN: /* person */
393 case A_RO: /* role */
394 case A_MR: /* mbrs-by-ref */
395 case A_MB: /* mnt-by */
396 case A_ML: /* mnt-lower */
397 case A_MO: /* member-of */
398 case A_SD: /* sub-dom */
399 case A_RZ: /* rev-srv */
400 case A_NS: /* nserver */
401 parse->new_attr_list = ud_split_attribute(parse->new_attr_list, attr);
402 break;
403 default:
404 parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
405 break;
406 }
407
408 }
409
410
411 /******************************************************************
412 * int ud_normalize() *
413 * *
414 * function accepts Obj_parse_t structure with attribute list *
415 * *
416 * It makes another pass to normalize object *
417 * Normalization includes: *
418 * *
419 * 1) reorder attributes *
420 * . mnt-by should go before member-of to allow correct *
421 * membership autorization (still done in RIPupd) *
422 * . nic-hdl should go before any admin-c, tech-c to prevent *
423 * errrors in self referencing role objects *
424 * 2) in case of pn/ro check nic handle and replace it in *
425 * the obj->object if needed (AUTO) *
426 * 3) split some attributes that allow lists *
427 ******************************************************************/
428 static int ud_normalize(Obj_parse_t *parse)
/* [<][>][^][v][top][bottom][index][help] */
429 {
430 GSList *old_attr_list;
431 GSList *first_element;
432 Attribute_t *class_attr;
433
434 old_attr_list = parse->obj->attributes;
435 /* get class attribute - the first one */
436 first_element = g_slist_nth(old_attr_list, 0);
437 class_attr = (Attribute_t *)first_element->data;
438 /* save object name for reporting and checking ref integrity for names (legacy) */
439 parse->object_name = g_strdup(class_attr->value);
440 /* reorder the attributes mnt-by and nic-hdl come first */
441 /*g_slist_foreach(old_attr_list, each_attribute_1_pass, parse);
442 g_slist_free(old_attr_list);
443 old_attr_list = parse->new_attr_list; */
444 parse->new_attr_list = NULL;
445 /* assign auto nic-hdl and slit attributes */
446 g_slist_foreach(old_attr_list, each_attribute_2_pass, parse);
447 g_slist_free(old_attr_list);
448 parse->obj->attributes = parse->new_attr_list;
449
450 return(1);
451 }
452
453
454 /******************************************************************
455 * report_transaction() *
456 * *
457 * Prints error report to the log *
458 * *
459 * reason - additional message that will be included *
460 * *
461 * *****************************************************************/
462 static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason)
/* [<][>][^][v][top][bottom][index][help] */
463 {
464 int result=0;
465 ut_timer_t fotime;
466 float timediff;
467 const char *class_name = DF_class_type2name(tr->class_type);
468 char *primary_key = tr->K->str;
469
470
471 /* calculate statistics */
472 UT_timeget(&fotime);
473 timediff = UT_timediff(psotime, &fotime);
474
475 if(tr->succeeded==0) {
476 result=tr->error;
477 log->num_failed++;
478 ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] %.2fs FAILED [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
479 /* ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] object: FAILED [%s][%s](%d/%d)", transaction_id, , reason, log->num_failed, (log->num_failed)+(log->num_ok)); */
480 if(result & ERROR_U_OBJ) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: referential integrity error", transaction_id);
481 if(result & ERROR_U_AUT) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: authentication error", transaction_id);
482 if(result & ERROR_U_BADOP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: unsupported operation", transaction_id);
483 if(result & ERROR_U_COP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: conflicting operation", transaction_id);
484 if(result & ERROR_U_NSUP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: this type is not supported", transaction_id);
485 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
486 result=1; /* # of failures */
487 }
488 else {
489 result=0;
490 log->num_ok++;
491 /* ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: OK [%s](%d/%d)", transaction_id, obj_name, log->num_ok, (log->num_failed)+(log->num_ok)); */
492 ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] %.2fs OK [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
493 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
494 }
495
496 return(result);
497 }/* report_transaction() */
498
499
500
501 /************************************************************
502 * process_nrtm() *
503 * *
504 * Process object in NRTM client mode *
505 * *
506 * nrtm - pointer to _nrtm structure *
507 * log - pointer to Log_t structure *
508 * object_name - name of the object *
509 * operation - operation code (OP_ADD/OP_DEL) *
510 * *
511 * Returns: *
512 * 1 - okay *
513 * <0 - error *
514 * *
515 ************************************************************/
516
517 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
/* [<][>][^][v][top][bottom][index][help] */
518 {
519 int result=0;
520 int dummy=0;
521 struct _nrtm *nrtm = ud_stream->nrtm;
522 long serial_id;
523 Log_t *log_ptr= &(ud_stream->log);
524 ut_timer_t sotime;
525 int ta_upd_nhr;
526
527 /* Start timer for statistics */
528 UT_timeget(&sotime);
529
530 /* We allow NRTM updates for some inconsistent objects */
531 /* One of the examples is reference by name which looks like nic-handle */
532 /* For this purpose we allow dummy creation when updating an object */
533 /* We also check for dummy allowance when deleting an object */
534 /* this is done to allow deletion of person objects referenced by name */
535
536 tr->mode|=B_DUMMY;
537 if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
538
539 switch (operation) {
540
541 case OP_ADD:
542 if(nrtm->tr){ /* DEL ADD => saved*/
543 if(tr->object_id==0) {
544 /* object does not exist in the DB */
545 /* delete the previous(saved) object*/
546 object_process(nrtm->tr);
547 /* create DEL serial */
548 UD_lock_serial(nrtm->tr);
549 serial_id = UD_create_serial(nrtm->tr);
550 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
551 UD_commit_serial(nrtm->tr);
552 UD_unlock_serial(nrtm->tr);
553 /* Mark TR as clean */
554 TR_mark_clean(nrtm->tr);
555 /* log the transaction */
556 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
557
558 object_free(nrtm->tr->object);
559 transaction_free(nrtm->tr); nrtm->tr=NULL;
560
561 /* Create an object and update NHR */
562 tr->action=(TA_CREATE | ta_upd_nhr);
563 /* restart the timer for statistics */
564 UT_timeget(&sotime);
565 object_process(tr); /* create a new one*/
566 /* create ADD serial */
567 UD_lock_serial(tr);
568 serial_id = UD_create_serial(tr);
569 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
570 UD_commit_serial(tr);
571 UD_unlock_serial(tr);
572 /* Mark TR as clean */
573 TR_mark_clean(tr);
574 /* log the transaction */
575 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
576 }
577 else {
578 /* object already exists in the DB - update or dummy replacement*/
579 /*compare the two, may be we may collapse operations*/
580 if(tr->object_id==nrtm->tr->object_id) {
581 /* DEL-ADD ->> UPDATE */
582 object_free(nrtm->tr->object);
583 transaction_free(nrtm->tr); nrtm->tr=NULL;
584 tr->action=TA_UPD_CLLPS;
585 object_process(tr);
586 /* create DEL+ADD serial records */
587 UD_lock_serial(tr);
588 tr->action=TA_DELETE; serial_id = UD_create_serial(tr);
589 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL(UPD)");
590
591 /* restart the timer for statistics */
592 UT_timeget(&sotime);
593 tr->sequence_id++;
594 tr->action=TA_CREATE; serial_id = UD_create_serial(tr);
595 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
596 UD_commit_serial(tr);
597 UD_unlock_serial(tr);
598 /* Mark TR as clean */
599 TR_mark_clean(tr);
600 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD(UPD)");
601 }
602 else { /* this should be a dummy object in the database(that we are going to replace with the real one */
603 /* or an interleaved operation*/
604 object_process(nrtm->tr); /* delete the previous(saved) object*/
605 /* create a DEL serial record */
606 UD_lock_serial(nrtm->tr);
607 serial_id = UD_create_serial(nrtm->tr);
608 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
609 UD_commit_serial(nrtm->tr);
610 UD_unlock_serial(nrtm->tr);
611 /* Mark TR as clean */
612 TR_mark_clean(nrtm->tr);
613 /* result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");*/
614 /* log the transaction */
615 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
616
617
618 object_free(nrtm->tr->object);
619 transaction_free(nrtm->tr); nrtm->tr=NULL;
620
621 /* restart the timer for statistics */
622 UT_timeget(&sotime);
623
624 tr->action=TA_UPDATE;
625 /* check if we are replacing a dummy object */
626 dummy=isdummy(tr);
627 /* If we are replacing dummy with a real object update NHR */
628 if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
629 /* fprintf(stderr,"UPDATE next(dummy)\n"); */
630 object_process(tr); /* create a new one*/
631 /* result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new"); */
632 /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
633 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
634 /* create ADD serial record */
635 UD_lock_serial(tr);
636 serial_id = UD_create_serial(tr);
637 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
638 UD_commit_serial(tr);
639 UD_unlock_serial(tr);
640 /* Mark TR as clean */
641 TR_mark_clean(tr);
642 /* log the transaction */
643 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
644
645 }
646 }
647 }
648 else { /* ADD ADD =>brand new object*/
649 if(tr->object_id==0) {
650 /* fprintf(stderr,"CREATE new\n");*/
651 /* Create an object and update NHR */
652 tr->action=(TA_CREATE | ta_upd_nhr);
653 object_process(tr);
654 /* create ADD serial */
655 UD_lock_serial(tr);
656 serial_id = UD_create_serial(tr);
657 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
658 UD_commit_serial(tr);
659 UD_unlock_serial(tr);
660
661 /* Mark TR as clean */
662 TR_mark_clean(tr);
663 /* log the transaction */
664 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
665 }
666 else { /* object already exists in the database */
667 /* this may happen because of dummies*/
668 /* or with some implementations of mirroring protocol that have atomic update */
669 /* instead of add + del */
670 tr->action=TA_UPDATE;
671 dummy=isdummy(tr);
672 /* If we are replacing dummy with a real object update NHR */
673 if(dummy==1) tr->action = ( ta_upd_nhr| TA_UPD_DUMMY);
674 object_process(tr);
675 /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
676 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
677 /* create ADD serial record */
678 UD_lock_serial(tr);
679 serial_id = UD_create_serial(tr);
680 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
681 UD_commit_serial(tr);
682 UD_unlock_serial(tr);
683 /* Mark TR as clean */
684 TR_mark_clean(tr);
685 /* log the transaction */
686 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
687
688 }
689 }
690 break;
691
692 case OP_DEL:
693 if(nrtm->tr){ /*DEL DEL =>saved */
694 object_process(nrtm->tr); /* delete the previous(saved) object*/
695 /* create DEL serial record */
696 UD_lock_serial(nrtm->tr);
697 serial_id = UD_create_serial(nrtm->tr);
698 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
699 UD_commit_serial(nrtm->tr);
700 UD_unlock_serial(nrtm->tr);
701 /* Mark TR as clean */
702 TR_mark_clean(nrtm->tr);
703 /* log the transaction */
704 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
705
706 object_free(nrtm->tr->object);
707 transaction_free(nrtm->tr); nrtm->tr=NULL;
708 }
709 /* save the real object (not a dummy one ) */
710 if(tr->object_id>0 && !isdummy(tr)){
711 /* save the object*/
712 tr->action=(TA_DELETE | ta_upd_nhr);
713 nrtm->tr=tr;
714 return(0);
715 }
716 else { /* this is an error - Trying to DEL non-existing object*/
717 tr->succeeded=0; tr->error|=ERROR_U_COP;
718 tr->action=(TA_DELETE | ta_upd_nhr);
719 /* create and initialize TR record for crash recovery */
720 TR_create_record(tr);
721 /* create DEL serial record anyway */
722 UD_lock_serial(tr);
723 serial_id = UD_create_serial(tr);
724 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
725 UD_commit_serial(tr);
726 UD_unlock_serial(tr);
727 /* Mark TR as clean */
728 TR_mark_clean(tr);
729 /* log the transaction */
730 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL: non-existing object");
731
732 }
733 break;
734
735 default:
736 tr->succeeded=0; tr->error |=ERROR_U_BADOP;
737 break;
738 }
739
740 /* Free resources */
741 object_free(tr->object);
742 transaction_free(tr);
743
744 return(result);
745 } /* process_nrtm() */
746
747
748
749 /************************************************************
750 * process_updates() *
751 * *
752 * Process object in update mode *
753 * *
754 * ud_stream - pointer to UD_stream structure *
755 * object_name - name of the object *
756 * operation - operation code (OP_ADD/OP_DEL) *
757 * *
758 * Note: *
759 * Frees tr and tr->obj on exit *
760 * *
761 * Returns: *
762 * 0 - okay *
763 * <0- number of failed objects *
764 * *
765 ************************************************************/
766
767 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
/* [<][>][^][v][top][bottom][index][help] */
768 {
769 int result=0;
770 Log_t *log_ptr= &(ud_stream->log);
771 int dummy=0;
772 ut_timer_t sotime;
773 int ta_upd_nhr;
774 char *reason;
775
776 /* Start timer for statistics */
777 UT_timeget(&sotime);
778 if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
779
780 switch(operation) {
781 /* Compare operations and report an error if they do not match */
782 case OP_ADD:
783 if(tr->object_id!=0) { /* trying to create, but object exists */
784 tr->succeeded=0; tr->error|=ERROR_U_COP;
785 reason="U:ADD:object already exists";
786 UD_ack(tr); /* Send a NACK */
787 } else {
788 /* Action: create the object and update NHR */
789 tr->action=(TA_CREATE | ta_upd_nhr);
790 reason="U:ADD";
791 object_process(tr);
792 }
793 break;
794 case OP_UPD:
795 if(tr->object_id==0) { /* trying to update non-existing object*/
796 tr->succeeded=0; tr->error|=ERROR_U_COP;
797 reason="U:UPD:non-existing object";
798 UD_ack(tr); /* Send a NACK */
799 } else {
800 tr->action=TA_UPDATE;
801 reason="U:UPD";
802 dummy=isdummy(tr);
803 /* If we are replacing dummy with a real object update NHR */
804 if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
805 object_process(tr);
806 }
807 break;
808
809 case OP_DEL:
810 if(tr->object_id==0) { /* trying t delete non-existing object*/
811 tr->succeeded=0; tr->error|=ERROR_U_COP;
812 reason="U:DEL:non-existing object";
813 UD_ack(tr);
814 } else {
815 tr->action=(TA_DELETE | ta_upd_nhr);
816 reason="U:DEL";
817 object_process(tr);
818 }
819 break;
820
821 default:
822 /* bad operation for this mode if not standalone */
823 if(IS_STANDALONE(tr->mode)) {
824 if(tr->object_id==0){
825 tr->action=(TA_CREATE | ta_upd_nhr);
826 reason="U:ADD";
827 }
828 else {
829 tr->action=TA_UPDATE;
830 reason="U:UPD";
831 }
832 object_process(tr);
833 }
834 else {
835 tr->succeeded=0;
836 tr->error|=ERROR_U_BADOP;
837 reason="U:bad operation";
838 UD_ack(tr); /* Send a NACK */
839 }
840 break;
841 }
842 /* If not in standalone mode create serial and copy error transcript */
843 if(!IS_STANDALONE(tr->mode)) {
844 if(tr->succeeded){
845 /* we don't want to generate DEL serial for dummy replacement*/
846 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
847 UD_lock_serial(tr);
848 UD_create_serial(tr);
849 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
850 UD_commit_serial(tr);
851 UD_unlock_serial(tr);
852 /* Mark the TR as clean */
853 TR_mark_clean(tr);
854 }
855 }
856
857 /* Make a report. U stands for update stream. No reason */
858 result=report_transaction(tr, tr->transaction_id, log_ptr, &sotime, reason);
859
860 /* Free resources */
861 object_free(tr->object);
862 transaction_free(tr);
863
864 return(result);
865
866 } /* process_updates() */
867
868
869 /************************************************************
870 * *
871 * int process_transaction() *
872 * *
873 * Processes the transaction *
874 * *
875 * ud_stream - pointer to UD_stream_t structure *
876 * *
877 * Returns: *
878 * 0 - no error *
879 * <0- number of failed objects *
880 * *
881 ************************************************************/
882
883 /* It frees the obj */
884
885 static int process_transaction(UD_stream_t *ud_stream,
/* [<][>][^][v][top][bottom][index][help] */
886 Obj_parse_t *parse,
887 int operation,
888 long transaction_id)
889 {
890 Transaction_t *tr = NULL;
891 Object_t *obj = parse->obj;
892 int result;
893
894 /* check if the requested transaction has already been processed */
895 /* this may happen in case of crash. If so, just send an ack and return */
896 if(TR_check(ud_stream->db_connection, transaction_id, (ud_stream->condat).sock))return(1);
897
898 /* start new transaction now */
899 tr = transaction_new(ud_stream->db_connection, obj->type);
900
901 /* Return with error if transaction cannot be created */
902 if (tr == NULL) die;
903
904 tr->mode=ud_stream->ud_mode;
905 tr->load_pass=ud_stream->load_pass;
906 tr->object=obj;
907 tr->nh=parse->nh_ptr;
908 tr->source_hdl=ud_stream->source_hdl;
909 tr->socket=(ud_stream->condat).sock;
910 tr->transaction_id=transaction_id;
911
912 /* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */
913 if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; }
914
915 /* For the first load pass we only create objects */
916 if(ud_stream->load_pass==1) {
917 Object_t *obj=tr->object;
918 /* still we need to fill tr->K (last.pkey) field */
919 g_slist_foreach(obj->attributes, ud_each_primary_key_select, tr);
920 tr->object_id=0;
921 }
922 else tr->object_id=get_object_id(tr);
923
924 /* Object cannot be retrieved */
925 if(tr->object_id==-1) { /* DB error*/
926 tr->succeeded=0;
927 tr->error |= ERROR_U_DBS;
928 ER_perror(FAC_UD, UD_SQL, "%s: Object cannot be retrieved", parse->object_name);
929 die;
930 transaction_free(tr);
931 object_free(obj);
932 }
933 /* save the name of person/role as we need it for referential */
934 /* integrity check when deleting the object against names. */
935 /* This is needed to support legacy references by name rather */
936 /* then by nic_hdl */
937 if((tr->class_type==C_PN) || (tr->class_type==C_RO)){
938 /* Save the value */
939 tr->save=g_strdup(parse->object_name);
940 /* attribute_free(attr, NULL); */
941 }
942
943 /* Process transaction. tr and obj are freed inside the process_* functions */
944
945 if(IS_UPDATE(ud_stream->ud_mode))
946 /* We are in update mode */
947 result=process_updates(ud_stream, tr, operation);
948 else
949 /* We are in NRTM mode */
950 result=process_nrtm(ud_stream, tr, operation);
951
952 return(result);
953
954 }
955
956
957 /************************************************************
958 * *
959 * int UD_process_stream(UD_stream_t *ud_stream) *
960 * *
961 * Processes the stream *
962 * *
963 * ud_stream - pointer to UD_stream_t structure *
964 * *
965 * Returns: *
966 * in update mode (!standalone)(1 object processed): *
967 * 1 - no error *
968 * <0- errors *
969 * *
970 * in NRTM & standalone modes *
971 * total number of object processed *
972 * *
973 ************************************************************/
974
975 int UD_process_stream(UD_stream_t *ud_stream)
/* [<][>][^][v][top][bottom][index][help] */
976 {
977 char line_buff[STR_XXL];
978 Object_t *obj = NULL;
979 SQ_connection_t *sql_connection;
980 int start_object;
981 int a_type;
982 struct _nrtm *nrtm;
983 Log_t *log_ptr= &(ud_stream->log);
984 ut_timer_t stime, ftime, sotime;
985 float obj_second1, obj_second10, timediff;
986 int result;
987 int operation=0;
988 int interrupt=0;
989 int do_update;
990 int default_ud_mode = ud_stream->ud_mode;
991 Line_Type_t linetype;
992 Transaction_t *tr;
993 long transaction_id=0; /* transaction_id (to be supplied by DBupdate and stored in Database) */
994 long serial_id;
995 Obj_parse_t obj_parse; /* the structure used to parse a text object */
996
997
998
999
1000 nrtm=ud_stream->nrtm;
1001 start_object = 1;
1002 a_type=-1;
1003
1004
1005 /* Check connection to the database */
1006 if(SQ_ping(ud_stream->db_connection)) {
1007 ER_perror(FAC_UD, UD_SQL, "%s", SQ_error(ud_stream->db_connection));
1008 die;
1009 }
1010
1011 sql_connection=ud_stream->db_connection;
1012
1013 ud_parse_init(sql_connection, &obj_parse);
1014 /* Start timer for statistics */
1015 UT_timeget(&stime);
1016
1017 /* Main loop. Reading input stream line by line */
1018 /* Empty line signals to start processing an object, if we have it */
1019 while (SK_cd_gets(&ud_stream->condat, line_buff, sizeof(line_buff))>0) {
1020
1021
1022 switch (linetype=line_type(line_buff, &transaction_id)) {
1023 case LINE_ATTRIBUTE:
1024 /* parse the object line by line */
1025 obj = UD_parse_object(&obj_parse, line_buff);
1026
1027 break;
1028
1029 case LINE_COMMENT:
1030 break;
1031
1032 case LINE_EOF:
1033 break;
1034
1035 case LINE_ACK:
1036 tr = transaction_new(ud_stream->db_connection, 0);
1037 tr->transaction_id=transaction_id;
1038 TR_delete_record(tr);
1039 transaction_free(tr);
1040 break;
1041
1042
1043 case LINE_ADD:
1044 /* restore the default operation mode */
1045 operation=OP_ADD;
1046 ud_stream->ud_mode=default_ud_mode;
1047 break;
1048
1049 case LINE_OVERRIDE_ADD:
1050 /* for override - switch the dummy bit on */
1051 operation=OP_ADD;
1052 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1053 break;
1054
1055 case LINE_UPD:
1056 /* restore the default operation mode */
1057 operation=OP_UPD;
1058 ud_stream->ud_mode=default_ud_mode;
1059 break;
1060
1061 case LINE_OVERRIDE_UPD:
1062 /* for override - switch the dummy bit on */
1063 operation=OP_UPD;
1064 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1065 break;
1066
1067 case LINE_DEL:
1068 /* restore the default operation mode */
1069 operation=OP_DEL;
1070 ud_stream->ud_mode=default_ud_mode;
1071 break;
1072
1073 case LINE_OVERRIDE_DEL:
1074 /* for override - switch the dummy bit on */
1075 operation=OP_DEL;
1076 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1077 break;
1078
1079 case LINE_EMPTY:
1080 /* start processing the object */
1081 if ((obj=obj_parse.obj)) { /* if not just garbage*/
1082 /* normalize the object (reorder and split attributes */
1083 ud_normalize(&obj_parse);
1084 /* XXX */
1085 /* print_object(obj); */
1086
1087 /* start new transaction now */
1088 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: [%s] ", transaction_id, obj_parse.object_name);
1089
1090 /* fprintf(stderr, "transction # %ld\n", transaction_id); */
1091 result=process_transaction(ud_stream, &obj_parse, operation, transaction_id);
1092 /* process_transaction() frees tr and obj structures, */
1093 /* so make sure we'll not reference these objects in the future */
1094 operation=OP_NOOP;
1095 transaction_id=0;
1096 ud_stream->ud_mode=default_ud_mode;
1097 ud_parse_free(&obj_parse);
1098
1099 /* this is a good place for quick interrupt */
1100 do_update=CO_get_do_update();
1101 if (do_update) interrupt=0; else interrupt=1;
1102 } /* if this is a real object */
1103 /* initialize the parsing structure */
1104 ud_parse_init(sql_connection, &obj_parse);
1105
1106 break;
1107
1108 default:
1109 die;
1110 } /* switch */
1111
1112 /* Finish processing if interrupt has been set */
1113 if (interrupt) break;
1114 } /* Main loop of data stream processing : while */
1115
1116 /* Some postprocessing */
1117 if(IS_NRTM_CLNT(ud_stream->ud_mode)){
1118 /* We are in NRTM mode */
1119 /* Clean up */
1120 /* fclose(ud_stream->stream); */
1121 /* In NRTM mode there may be a saved object that is unprocessed */
1122 if(nrtm->tr){ /*saved backlog?*/
1123 /* restart the timer for statistics */
1124 UT_timeget(&sotime);
1125 object_process(nrtm->tr); /* delete the previous(saved) object*/
1126 /* result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name,
1127 "NRTM:DEL:While deleting previous(saved) object"); */
1128 /* create DEL serial record no matter what the result is */
1129 UD_lock_serial(nrtm->tr);
1130 serial_id = UD_create_serial(nrtm->tr);
1131 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
1132 UD_commit_serial(nrtm->tr);
1133 UD_unlock_serial(nrtm->tr);
1134 /* Mark TR as clean */
1135 TR_mark_clean(nrtm->tr);
1136 /* log the transaction */
1137 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
1138
1139 object_free(nrtm->tr->object);
1140 transaction_free(nrtm->tr); nrtm->tr=NULL;
1141 }
1142 }
1143
1144 /* That's all. Free GString */
1145 /* g_string_free(g_line_buff, TRUE);*/
1146
1147
1148 /* Calculate some statistics */
1149 /* ftime=time(NULL); */
1150 UT_timeget(&ftime);
1151 /* obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime);
1152 obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime); */
1153 timediff = UT_timediff(&stime, &ftime);
1154 obj_second1 = (float)(log_ptr->num_ok)/timediff;
1155 obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/timediff;
1156
1157 /* Print the report */
1158 if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) {
1159
1160 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s ******** report **********", UD_TAG);
1161 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects OK (%7.4f obj/s)", UD_TAG, log_ptr->num_ok, obj_second1);
1162 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects failed", UD_TAG, log_ptr->num_failed);
1163 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s average processing time %7.4f obj/s (%6.2f obj/min)", UD_TAG,
1164 obj_second10, obj_second10*60);
1165 result=log_ptr->num_ok+log_ptr->num_failed;
1166 }
1167 return(result);
1168
1169 } /* UD_process_stream */
1170