modules/ud/ud_process_stream.c
/* [<][>][^][v][top][bottom][index][help] */
FUNCTIONS
This source file includes following functions.
- ud_parse_init
- ud_parse_free
- line_continuation
- ud_replace_substring
- ud_split_attribute
- escape_apostrophes
- line_type
- UD_parse_object
- each_attribute_2_pass
- ud_normalize
- report_transaction
- process_nrtm
- process_updates
- process_transaction
- UD_process_stream
1 /***************************************
2 $Revision: 1.55 $
3
4 Functions to process data stream( file, network socket, etc.)
5
6 Status: NOT REVUED, NOT TESTED
7
8 Author(s): Chris Ottrey, Andrei Robachevsky
9
10 ******************/ /******************
11 Modification History:
12 andrei (17/01/2000) Created.
13 ******************/ /******************
14 Copyright (c) 2000,2001 RIPE NCC
15
16 All Rights Reserved
17
18 Permission to use, copy, modify, and distribute this software and its
19 documentation for any purpose and without fee is hereby granted,
20 provided that the above copyright notice appear in all copies and that
21 both that copyright notice and this permission notice appear in
22 supporting documentation, and that the name of the author not be
23 used in advertising or publicity pertaining to distribution of the
24 software without specific, written prior permission.
25
26 THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
27 ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
28 AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
29 DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
30 AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
31 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 ***************************************/
33
34 #include "rip.h"
35
36 #include <sys/types.h>
37 #include <sys/socket.h>
38 #include <netdb.h>
39 #include <arpa/inet.h>
40 #include <unistd.h>
41 #include <sys/stat.h>
42 #include <fcntl.h>
43 #include <string.h>
44
45
46 typedef enum _Line_Type_t {
47 LINE_ATTRIBUTE,
48 LINE_COMMENT,
49 LINE_EMPTY,
50 LINE_EOF,
51 LINE_ADD,
52 LINE_UPD,
53 LINE_DEL,
54 LINE_OVERRIDE_ADD,
55 LINE_OVERRIDE_UPD,
56 LINE_OVERRIDE_DEL,
57 LINE_ACK
58 } Line_Type_t;
59
60 /* Maximum number of objects(serials) we can consume at a time */
61 #define SBUNCH 1000
62
63 static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason);
64 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
65 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
66 static int process_transaction(UD_stream_t *ud_stream, Obj_parse_t *parse, int operation, long transaction_id);
67
68 static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr);
69 static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse);
70 static void ud_parse_free(Obj_parse_t *parse);
71 static int line_continuation(char *line);
72 static GString *escape_apostrophes(GString *text);
73 static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff);
74 static int ud_replace_substring(GString *gstring, char *old, char *new);
75 static int ud_normalize(Obj_parse_t *parse);
76
77 /* Delimiters that separate list members, both RPS(,) and legacy( ) */
78 #define ATTR_DELIMITERS " ,\t"
79
80
81 static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse){
/* [<][>][^][v][top][bottom][index][help] */
82 bzero(parse, sizeof(Obj_parse_t));
83 parse->garbage = 0;
84 parse->sql_connection = sql_connection;
85 }
86
87 static void ud_parse_free(Obj_parse_t *parse){
/* [<][>][^][v][top][bottom][index][help] */
88 UT_free(parse->nic);
89 UT_free(parse->object_name);
90 }
91
92
93 /******************************************************************
94 * int line_continuation() *
95 * *
96 * Checks if the line starts with one of line continuation signs *
97 * Returns 1 if this is line continuation, 0 otherwise *
98 * *
99 * ****************************************************************/
100 static int line_continuation(char *line)
/* [<][>][^][v][top][bottom][index][help] */
101 {
102 switch(*line) {
103 case ' ':
104 case '\t':
105 case '+':
106 return(1); /* these indicate line continuation */
107 default: return(0);
108 }
109
110 }
111
112 /******************************************************************
113 * int ud_replace_substring() *
114 * *
115 * replace a subsring in a string *
116 * Returns -1 if no match was found *
117 * *
118 * ****************************************************************/
119 static int ud_replace_substring(GString *gstring, char *old, char *new)
/* [<][>][^][v][top][bottom][index][help] */
120 {
121 int i, i0=-1, j=0;
122
123 for (i=0; ((i < gstring->len) && (old[j] != '\0')); i++){
124 if(gstring->str[i]==old[j]){
125 if(i0==-1)i0=i;
126 j++;
127 } else {
128 i0=-1;
129 j=0;
130 }
131 }
132 /* if we haven't reached the end of the old substring */
133 /* or no match occured return -1 */
134 if((old[j] != '\0') || (i0==-1)) return (-1);
135 g_string_erase(gstring, i0, j);
136 g_string_insert(gstring, i0, new);
137 return(0);
138 }
139
140 /************************************************************
141 * *
142 * The function to splits attribute value into multiple *
143 * words and appends them as attr_type - attr_valu pairs *
144 * to the attr_list *
145 * *
146 * *
147 ************************************************************/
148 static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr)
/* [<][>][^][v][top][bottom][index][help] */
149 {
150 char *token;
151 char *split;
152 char *value, *n;
153 Attribute_t *attr_split;
154 GSList *the_list = attr_list;
155
156
157 /* check for line continuation (+) */
158 if (strncmp(attr->value, "+", 1) == 0) attr->value++;
159 /* check for end-of-line comments */
160 n = index(attr->value, '#');
161 /* if there is no comment check for trailing \n */
162 if(n == NULL) n = index(attr->value, '\n');
163 /* now copy the clean value into the attribute */
164 if(n == NULL) value = g_strdup(attr->value);
165 else value = g_strndup(attr->value, (n - attr->value));
166
167 token=value;
168 while((split=strsep(&token, ATTR_DELIMITERS))){
169 if (*split != '\0'){
170 attr_split = attribute_new1(attr->type, split);
171 if (attr_split) the_list = g_slist_append(the_list, attr_split);
172 }
173 }
174 UT_free(value);
175 attribute_free(attr, NULL);
176
177 return(the_list);
178 }
179
180 /* XXX */
181 /* commented out because it is unused - shane
182 static void each_attribute_print(void *element_data, void *tr_ptr)
183 {
184
185 Attribute_t *attr = (Attribute_t *)element_data;
186
187 fprintf(stderr, "[%d|%s]\n", attr->type, attr->value);
188
189 }
190 */
191
192 /* XXX */
193 /* commented out because it is unused - shane
194 static void print_object(Object_t *obj)
195 {
196 g_slist_foreach(obj->attributes, each_attribute_print, NULL);
197 fprintf(stderr, ">>>>>\n%s\n", obj->object->str);
198 }
199 */
200
201
202 /******************************************************************
203 * GString *escape_apostrophes() *
204 * Escapes apostrophes in the text so they do not confuse printf *
205 * functions and don't corrupt SQL queries *
206 * *
207 * *****************************************************************/
208 static GString *escape_apostrophes(GString *text) {
/* [<][>][^][v][top][bottom][index][help] */
209 int i;
210 for (i=0; i < text->len; i++) {
211 if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
212 text = g_string_insert_c(text, i, '\\');
213 i++;
214 }
215 }
216 return(text);
217 } /* escape_apostrophes() */
218
219
220 /******************************************************************
221 * Line_Type_t line_type(e) *
222 * Determines the line type analysing the first letters *
223 * *
224 * ****************************************************************/
225 static Line_Type_t line_type(const char *line, long *transaction_id) {
/* [<][>][^][v][top][bottom][index][help] */
226
227 if (strncmp(line, "# EOF", 4) == 0) return(LINE_EOF);
228 if (strncmp(line, "#", 1) == 0) return(LINE_COMMENT);
229 if (strcmp(line, "\n") == 0) return(LINE_EMPTY);
230
231 if (strncmp(line, "ACK", 3) == 0) {
232 *transaction_id = atol(line+3);
233 return(LINE_ACK);
234 }
235 if (strncmp(line, "ADD_OVERRIDE", 12) == 0) {
236 *transaction_id = atol(line+12);
237 return(LINE_OVERRIDE_ADD);
238 }
239 if (strncmp(line, "UPD_OVERRIDE", 12) == 0) {
240 *transaction_id = atol(line+12);
241 return(LINE_OVERRIDE_UPD);
242 }
243 if (strncmp(line, "DEL_OVERRIDE", 12) == 0) {
244 *transaction_id = atol(line+12);
245 return(LINE_OVERRIDE_DEL);
246 }
247
248 if (strncmp(line, "ADD", 3) == 0) {
249 *transaction_id = atol(line+3);
250 return(LINE_ADD);
251 }
252 if (strncmp(line, "UPD", 3) == 0) {
253 *transaction_id = atol(line+3);
254 return(LINE_UPD);
255 }
256 if (strncmp(line, "DEL", 3) == 0) {
257 *transaction_id = atol(line+3);
258 return(LINE_DEL);
259 }
260
261 /* Otherwise this is an attribute */
262 return(LINE_ATTRIBUTE);
263
264 } /* line_type() */
265
266 /******************************************************************
267 * Object_t *UD_parse_object() *
268 * *
269 * Parses the object accepting line by line *
270 * Stores the object and list of attributes in Obj_parse_t struct *
271 * *
272 * ****************************************************************/
273 static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff)
/* [<][>][^][v][top][bottom][index][help] */
274 {
275 GString *g_line_buff;
276 Attribute_t *attr;
277
278 /* in case of garbage */
279 if(parse->garbage) return(NULL);
280
281 if(parse->obj==NULL) {
282 /* which means this is the start - create a new object*/
283 parse->obj = object_new(line_buff);
284 /* if this is not a class attribute - this is a garbage till the next blank line */
285 if(parse->obj==NULL) {
286 parse->garbage = 1;
287 return(NULL);
288 }
289 }
290
291 /* allocate buffer or the string */
292 if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){
293 ER_perror(FAC_UD, UD_MEM, "cannot allocate gstring");
294 die;
295 }
296 /* copy the input */
297 g_string_sprintf(g_line_buff, "%s", line_buff);
298 /* escape apostrophes in the input line */
299 g_line_buff=escape_apostrophes(g_line_buff);
300
301 attr = attribute_new(g_line_buff->str);
302 if (attr){
303 /* save this attribute for future line continuations */
304 /* skip attributes that are not stored in SQL database */
305 if(DF_get_update_query_type(attr->type)==UD_NULL_){
306 parse->current_attr = NULL;
307 attribute_free(attr, NULL);
308 }
309 else {
310 parse->current_attr = attr;
311 if((attr->type == A_MB) || (attr->type == A_NH)) {
312 /* insert it at the beginning */
313 /* mnt-by should go before member-of to allow correct membership autorization (still done in RIPupd) */
314 /* nic-hdl should go before any admin-c, tech-c to prevent errrors in self referencing role objects */
315 parse->obj->attributes = g_slist_insert(parse->obj->attributes, parse->current_attr, 1);
316 } else {
317 /* append it to the attribute list */
318 parse->obj->attributes = g_slist_append(parse->obj->attributes, parse->current_attr);
319 }
320 }
321 }
322 /* if this is not a valid attribute, then this may be unknown attr or line continuation */
323 else
324 if(line_continuation(g_line_buff->str))
325 {
326 /* if parse->attr == NULL, then this attr is not stored in SQL database - skip it */
327 if(parse->current_attr){
328 /* this is continuation of a previous attribute */
329 char *line_value, *new_value;
330 char *n;
331 /* normalize and append to current_attr->value */
332 /* check for end-of-line comments */
333 n = index(g_line_buff->str, '#');
334 /* if there is no comment check for trailing \n */
335 if(n == NULL) n = index(g_line_buff->str, '\n');
336 /* now copy the clean value into the attribute */
337 if(n == NULL) line_value = g_strdup(g_line_buff->str);
338 else line_value = g_strndup(g_line_buff->str, (n - g_line_buff->str));
339 /* replace line continuation character with ' ' */
340 *line_value = ' ';
341 new_value = g_strconcat(parse->current_attr->value, line_value, NULL);
342 parse->current_attr = attribute_upd(parse->current_attr, parse->current_attr->type, new_value);
343 UT_free(new_value);
344 UT_free(line_value);
345 }
346 }
347 /* otherwise, this is unknown attribute, so we need to break the continuation */
348 else parse->current_attr=NULL;
349 /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */
350 g_string_sprintfa(parse->obj->object, "%s", g_line_buff->str);
351 g_string_free(g_line_buff, TRUE);
352 return(parse->obj);
353 }
354
355 /******************************************************************
356 * int each_attribute_2_pass() *
357 * *
358 * Assigns nic-handles in case of AUTO *
359 * Splits attributes *
360 * * *
361 ******************************************************************/
362 void each_attribute_2_pass(void *element_data, void *ptr)
/* [<][>][^][v][top][bottom][index][help] */
363 {
364 Attribute_t *attr = (Attribute_t *)element_data;
365 Obj_parse_t *parse = (Obj_parse_t *)ptr;
366
367
368 /* Strip the white space */
369 g_strstrip(attr->value);
370
371 switch(attr->type){
372 case A_NH: /* nic-hdl */
373 /* Parse the string into nh structure */
374 /* In case of an AUTO NIC handle check the ID in the database */
375 /* Possible errors leave to core processing */
376 if(NH_parse(attr->value, &parse->nh_ptr) == 0) {
377 /* this is an AUTO nic handle */
378 /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsing nic handle: [%s]", UD_TAG, attr->value);*/
379 /* Check if we can allocate it */
380 if(NH_check(parse->nh_ptr, parse->sql_connection)>0){
381 /* Convert nh to the database format */
382 parse->nic = NH_convert(parse->nh_ptr);
383 /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsed and converted nic handle: [%s]", UD_TAG, nic); */
384 /* replace the auto nic with a real one */
385 if(ud_replace_substring(parse->obj->object, attr->value, parse->nic)==-1) {
386 ER_perror(FAC_UD, UD_BUG, "cannot replace auto nic handle");
387 die;
388 }
389 /* Update the attribute */
390 attr = attribute_upd(attr, attr->type, parse->nic);
391
392 }
393 }
394 parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
395 break;
396 case A_PN: /* person */
397 case A_RO: /* role */
398 case A_MR: /* mbrs-by-ref */
399 case A_MB: /* mnt-by */
400 case A_ML: /* mnt-lower */
401 case A_CT: /* cross-mnt */
402 case A_MO: /* member-of */
403 case A_SD: /* sub-dom */
404 case A_RZ: /* rev-srv */
405 case A_NS: /* nserver */
406 parse->new_attr_list = ud_split_attribute(parse->new_attr_list, attr);
407 break;
408 default:
409 parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
410 break;
411 }
412
413 }
414
415
416 /******************************************************************
417 * int ud_normalize() *
418 * *
419 * function accepts Obj_parse_t structure with attribute list *
420 * *
421 * It makes another pass to normalize object *
422 * Normalization includes: *
423 * *
424 * 1) reorder attributes *
425 * . mnt-by should go before member-of to allow correct *
426 * membership autorization (still done in RIPupd) *
427 * . nic-hdl should go before any admin-c, tech-c to prevent *
428 * errrors in self referencing role objects *
429 * 2) in case of pn/ro check nic handle and replace it in *
430 * the obj->object if needed (AUTO) *
431 * 3) split some attributes that allow lists *
432 ******************************************************************/
433 static int ud_normalize(Obj_parse_t *parse)
/* [<][>][^][v][top][bottom][index][help] */
434 {
435 GSList *old_attr_list;
436 GSList *first_element;
437 Attribute_t *class_attr;
438
439 old_attr_list = parse->obj->attributes;
440 /* get class attribute - the first one */
441 first_element = g_slist_nth(old_attr_list, 0);
442 class_attr = (Attribute_t *)first_element->data;
443 /* save object name for reporting and checking ref integrity for names (legacy) */
444 parse->object_name = g_strdup(class_attr->value);
445 /* reorder the attributes mnt-by and nic-hdl come first */
446 /*g_slist_foreach(old_attr_list, each_attribute_1_pass, parse);
447 g_slist_free(old_attr_list);
448 old_attr_list = parse->new_attr_list; */
449 parse->new_attr_list = NULL;
450 /* assign auto nic-hdl and slit attributes */
451 g_slist_foreach(old_attr_list, each_attribute_2_pass, parse);
452 g_slist_free(old_attr_list);
453 parse->obj->attributes = parse->new_attr_list;
454
455 return(1);
456 }
457
458
459 /******************************************************************
460 * report_transaction() *
461 * *
462 * Prints error report to the log *
463 * *
464 * reason - additional message that will be included *
465 * *
466 * *****************************************************************/
467 static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason)
/* [<][>][^][v][top][bottom][index][help] */
468 {
469 int result=0;
470 ut_timer_t fotime;
471 float timediff;
472 const char *class_name = DF_class_type2name(tr->class_type);
473 char *primary_key = tr->K->str;
474
475
476 /* calculate statistics */
477 UT_timeget(&fotime);
478 timediff = UT_timediff(psotime, &fotime);
479
480 if(tr->succeeded==0) {
481 result=tr->error;
482 log->num_failed++;
483 ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] %.2fs FAILED [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
484 /* ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] object: FAILED [%s][%s](%d/%d)", transaction_id, , reason, log->num_failed, (log->num_failed)+(log->num_ok)); */
485 if(result & ERROR_U_OBJ) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: referential integrity error", transaction_id);
486 if(result & ERROR_U_AUT) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: authentication error", transaction_id);
487 if(result & ERROR_U_BADOP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: unsupported operation", transaction_id);
488 if(result & ERROR_U_COP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: conflicting operation", transaction_id);
489 if(result & ERROR_U_NSUP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: this type is not supported", transaction_id);
490 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
491 result=1; /* # of failures */
492 }
493 else {
494 result=0;
495 log->num_ok++;
496 /* ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: OK [%s](%d/%d)", transaction_id, obj_name, log->num_ok, (log->num_failed)+(log->num_ok)); */
497 ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] %.2fs OK [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
498 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
499 }
500
501 return(result);
502 }/* report_transaction() */
503
504
505
506 /************************************************************
507 * process_nrtm() *
508 * *
509 * Process object in NRTM client mode *
510 * *
511 * nrtm - pointer to _nrtm structure *
512 * log - pointer to Log_t structure *
513 * object_name - name of the object *
514 * operation - operation code (OP_ADD/OP_DEL) *
515 * *
516 * Returns: *
517 * 1 - okay *
518 * <0 - error *
519 * *
520 ************************************************************/
521
522 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
/* [<][>][^][v][top][bottom][index][help] */
523 {
524 int result=0;
525 int dummy=0;
526 struct _nrtm *nrtm = ud_stream->nrtm;
527 long serial_id;
528 Log_t *log_ptr= &(ud_stream->log);
529 ut_timer_t sotime;
530 int ta_upd_nhr;
531
532 /* Start timer for statistics */
533 UT_timeget(&sotime);
534
535 /* We allow NRTM updates for some inconsistent objects */
536 /* One of the examples is reference by name which looks like nic-handle */
537 /* For this purpose we allow dummy creation when updating an object */
538 /* We also check for dummy allowance when deleting an object */
539 /* this is done to allow deletion of person objects referenced by name */
540
541 tr->mode|=B_DUMMY;
542 if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
543
544 switch (operation) {
545
546 case OP_ADD:
547 if(nrtm->tr){ /* DEL ADD => saved*/
548 if(tr->object_id==0) {
549 /* object does not exist in the DB */
550 /* delete the previous(saved) object*/
551 object_process(nrtm->tr);
552 /* create DEL serial */
553 UD_lock_serial(nrtm->tr);
554 serial_id = UD_create_serial(nrtm->tr);
555 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
556 UD_commit_serial(nrtm->tr);
557 UD_unlock_serial(nrtm->tr);
558 /* Mark TR as clean */
559 TR_mark_clean(nrtm->tr);
560 /* log the transaction */
561 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
562
563 object_free(nrtm->tr->object);
564 transaction_free(nrtm->tr); nrtm->tr=NULL;
565
566 /* Create an object and update NHR */
567 tr->action=(TA_CREATE | ta_upd_nhr);
568 /* restart the timer for statistics */
569 UT_timeget(&sotime);
570 object_process(tr); /* create a new one*/
571 /* create ADD serial */
572 UD_lock_serial(tr);
573 serial_id = UD_create_serial(tr);
574 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
575 UD_commit_serial(tr);
576 UD_unlock_serial(tr);
577 /* Mark TR as clean */
578 TR_mark_clean(tr);
579 /* log the transaction */
580 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
581 }
582 else {
583 /* object already exists in the DB - update or dummy replacement*/
584 /*compare the two, may be we may collapse operations*/
585 if(tr->object_id==nrtm->tr->object_id) {
586 /* DEL-ADD ->> UPDATE */
587 object_free(nrtm->tr->object);
588 transaction_free(nrtm->tr); nrtm->tr=NULL;
589 tr->action=TA_UPD_CLLPS;
590 object_process(tr);
591 /* create DEL+ADD serial records */
592 UD_lock_serial(tr);
593 tr->action=TA_DELETE; serial_id = UD_create_serial(tr);
594 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL(UPD)");
595
596 /* restart the timer for statistics */
597 UT_timeget(&sotime);
598 tr->sequence_id++;
599 tr->action=TA_CREATE; serial_id = UD_create_serial(tr);
600 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
601 UD_commit_serial(tr);
602 UD_unlock_serial(tr);
603 /* Mark TR as clean */
604 TR_mark_clean(tr);
605 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD(UPD)");
606 }
607 else { /* this should be a dummy object in the database(that we are going to replace with the real one */
608 /* or an interleaved operation*/
609 object_process(nrtm->tr); /* delete the previous(saved) object*/
610 /* create a DEL serial record */
611 UD_lock_serial(nrtm->tr);
612 serial_id = UD_create_serial(nrtm->tr);
613 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
614 UD_commit_serial(nrtm->tr);
615 UD_unlock_serial(nrtm->tr);
616 /* Mark TR as clean */
617 TR_mark_clean(nrtm->tr);
618 /* result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");*/
619 /* log the transaction */
620 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
621
622
623 object_free(nrtm->tr->object);
624 transaction_free(nrtm->tr); nrtm->tr=NULL;
625
626 /* restart the timer for statistics */
627 UT_timeget(&sotime);
628
629 tr->action=TA_UPDATE;
630 /* check if we are replacing a dummy object */
631 dummy=isdummy(tr);
632 /* If we are replacing dummy with a real object update NHR */
633 /* if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY); */
634 if(dummy==1) tr->action = TA_UPD_DUMMY;
635 /* fprintf(stderr,"UPDATE next(dummy)\n"); */
636 object_process(tr); /* create a new one*/
637 /* result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new"); */
638 /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
639 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
640 /* create ADD serial record */
641 UD_lock_serial(tr);
642 serial_id = UD_create_serial(tr);
643 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
644 UD_commit_serial(tr);
645 UD_unlock_serial(tr);
646 /* Mark TR as clean */
647 TR_mark_clean(tr);
648 /* log the transaction */
649 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
650
651 }
652 }
653 }
654 else { /* ADD ADD =>brand new object*/
655 if(tr->object_id==0) {
656 /* fprintf(stderr,"CREATE new\n");*/
657 /* Create an object and update NHR */
658 tr->action=(TA_CREATE | ta_upd_nhr);
659 object_process(tr);
660 /* create ADD serial */
661 UD_lock_serial(tr);
662 serial_id = UD_create_serial(tr);
663 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
664 UD_commit_serial(tr);
665 UD_unlock_serial(tr);
666
667 /* Mark TR as clean */
668 TR_mark_clean(tr);
669 /* log the transaction */
670 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
671 }
672 else { /* object already exists in the database */
673 /* this may happen because of dummies*/
674 /* or with some implementations of mirroring protocol that have atomic update */
675 /* instead of add + del */
676 tr->action=TA_UPDATE;
677 dummy=isdummy(tr);
678 /* If we are replacing dummy with a real object update NHR */
679 /* if(dummy==1) tr->action = ( ta_upd_nhr| TA_UPD_DUMMY);*/
680 if(dummy==1) tr->action = TA_UPD_DUMMY;
681 object_process(tr);
682 /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
683 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
684 /* create ADD serial record */
685 UD_lock_serial(tr);
686 serial_id = UD_create_serial(tr);
687 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
688 UD_commit_serial(tr);
689 UD_unlock_serial(tr);
690 /* Mark TR as clean */
691 TR_mark_clean(tr);
692 /* log the transaction */
693 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
694
695 }
696 }
697 break;
698
699 case OP_DEL:
700 if(nrtm->tr){ /*DEL DEL =>saved */
701 /* check this is not a deletion of the same object twise */
702 /* this should not happen but we cannot trust foreign sources */
703 /* in such case process saved transaction but fail the current one */
704 if(nrtm->tr->object_id == tr->object_id) tr->object_id=0;
705
706 object_process(nrtm->tr); /* delete the previous(saved) object*/
707 /* create DEL serial record */
708 UD_lock_serial(nrtm->tr);
709 serial_id = UD_create_serial(nrtm->tr);
710 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
711 UD_commit_serial(nrtm->tr);
712 UD_unlock_serial(nrtm->tr);
713 /* Mark TR as clean */
714 TR_mark_clean(nrtm->tr);
715 /* log the transaction */
716 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
717
718 object_free(nrtm->tr->object);
719 transaction_free(nrtm->tr); nrtm->tr=NULL;
720 }
721 /* save the real object (not a dummy one ) */
722 if(tr->object_id>0 && !isdummy(tr)){
723 /* save the object*/
724 tr->action=(TA_DELETE | ta_upd_nhr);
725 nrtm->tr=tr;
726 return(0);
727 }
728 else { /* this is an error - Trying to DEL non-existing object*/
729 tr->succeeded=0; tr->error|=ERROR_U_COP;
730 tr->action=(TA_DELETE | ta_upd_nhr);
731 /* create and initialize TR record for crash recovery */
732 TR_create_record(tr);
733 /* create DEL serial record anyway */
734 UD_lock_serial(tr);
735 serial_id = UD_create_serial(tr);
736 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
737 UD_commit_serial(tr);
738 UD_unlock_serial(tr);
739 /* Mark TR as clean */
740 TR_mark_clean(tr);
741 /* log the transaction */
742 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL: non-existing object");
743
744 }
745 break;
746
747 default:
748 tr->succeeded=0; tr->error |=ERROR_U_BADOP;
749 break;
750 }
751
752 /* Free resources */
753 object_free(tr->object);
754 transaction_free(tr);
755
756 return(result);
757 } /* process_nrtm() */
758
759
760
761 /************************************************************
762 * process_updates() *
763 * *
764 * Process object in update mode *
765 * *
766 * ud_stream - pointer to UD_stream structure *
767 * object_name - name of the object *
768 * operation - operation code (OP_ADD/OP_DEL) *
769 * *
770 * Note: *
771 * Frees tr and tr->obj on exit *
772 * *
773 * Returns: *
774 * 0 - okay *
775 * <0- number of failed objects *
776 * *
777 ************************************************************/
778
779 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
/* [<][>][^][v][top][bottom][index][help] */
780 {
781 int result=0;
782 Log_t *log_ptr= &(ud_stream->log);
783 int dummy=0;
784 ut_timer_t sotime;
785 int ta_upd_nhr;
786 char *reason;
787
788 /* Start timer for statistics */
789 UT_timeget(&sotime);
790 if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
791
792 switch(operation) {
793 /* Compare operations and report an error if they do not match */
794 case OP_ADD:
795 if(tr->object_id!=0) { /* trying to create, but object exists */
796 tr->succeeded=0; tr->error|=ERROR_U_COP;
797 reason="U:ADD:object already exists";
798 g_string_sprintfa(tr->error_script,"E[%d]:NEW requested but object already exists\n" ,ERROR_U_COP);
799 UD_ack(tr); /* Send a NACK */
800 } else {
801 /* Action: create the object and update NHR */
802 tr->action=(TA_CREATE | ta_upd_nhr);
803 reason="U:ADD";
804 object_process(tr);
805 }
806 break;
807 case OP_UPD:
808 if(tr->object_id==0) { /* trying to update non-existing object*/
809 tr->succeeded=0; tr->error|=ERROR_U_COP;
810 reason="U:UPD:non-existing object";
811 g_string_sprintfa(tr->error_script,"E[%d]:UPD requested but no existing object found\n" ,ERROR_U_COP);
812 UD_ack(tr); /* Send a NACK */
813 } else {
814 tr->action=TA_UPDATE;
815 reason="U:UPD";
816 dummy=isdummy(tr);
817 /* If we are replacing dummy with a real object update NHR */
818 /* if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY); */
819 if(dummy==1) tr->action = TA_UPD_DUMMY;
820 object_process(tr);
821 }
822 break;
823
824 case OP_DEL:
825 if(tr->object_id==0) { /* trying t delete non-existing object*/
826 tr->succeeded=0; tr->error|=ERROR_U_COP;
827 reason="U:DEL:non-existing object";
828 g_string_sprintfa(tr->error_script,"E[%d]:DEL requested but no existing object found\n" ,ERROR_U_COP);
829 UD_ack(tr);
830 } else {
831 tr->action=(TA_DELETE | ta_upd_nhr);
832 reason="U:DEL";
833 object_process(tr);
834 }
835 break;
836
837 default:
838 /* bad operation for this mode if not standalone */
839 if(IS_STANDALONE(tr->mode)) {
840 if(tr->object_id==0){
841 tr->action=(TA_CREATE | ta_upd_nhr);
842 reason="U:ADD";
843 }
844 else {
845 tr->action=TA_UPDATE;
846 reason="U:UPD";
847 }
848 object_process(tr);
849 }
850 else {
851 tr->succeeded=0;
852 tr->error|=ERROR_U_BADOP;
853 g_string_sprintfa(tr->error_script,"E[%d]:Unknown operation requested\n" ,ERROR_U_BADOP);
854 reason="U:bad operation";
855 UD_ack(tr); /* Send a NACK */
856 }
857 break;
858 }
859 /* If not in standalone mode create serial and copy error transcript */
860 if(!IS_STANDALONE(tr->mode)) {
861 if(tr->succeeded){
862 /* we don't want to generate DEL serial for dummy replacement*/
863 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
864 UD_lock_serial(tr);
865 UD_create_serial(tr);
866 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
867 UD_commit_serial(tr);
868 UD_unlock_serial(tr);
869 /* Mark the TR as clean */
870 TR_mark_clean(tr);
871 }
872 }
873
874 /* Make a report. U stands for update stream. No reason */
875 result=report_transaction(tr, tr->transaction_id, log_ptr, &sotime, reason);
876
877 /* Free resources */
878 object_free(tr->object);
879 transaction_free(tr);
880
881 return(result);
882
883 } /* process_updates() */
884
885
886 /************************************************************
887 * *
888 * int process_transaction() *
889 * *
890 * Processes the transaction *
891 * *
892 * ud_stream - pointer to UD_stream_t structure *
893 * *
894 * Returns: *
895 * 0 - no error *
896 * <0- number of failed objects *
897 * *
898 ************************************************************/
899
900 /* It frees the obj */
901
902 static int process_transaction(UD_stream_t *ud_stream,
/* [<][>][^][v][top][bottom][index][help] */
903 Obj_parse_t *parse,
904 int operation,
905 long transaction_id)
906 {
907 Transaction_t *tr = NULL;
908 Object_t *obj = parse->obj;
909 int result;
910
911 /* check if the requested transaction has already been processed */
912 /* this may happen in case of crash. If so, just send an ack and return */
913 if(TR_check(ud_stream->db_connection, transaction_id, (ud_stream->condat).sock))return(1);
914
915 /* start new transaction now */
916 tr = transaction_new(ud_stream->db_connection, obj->type);
917
918 /* Return with error if transaction cannot be created */
919 if (tr == NULL) die;
920
921 tr->mode=ud_stream->ud_mode;
922 tr->load_pass=ud_stream->load_pass;
923 tr->object=obj;
924 tr->nh=parse->nh_ptr;
925 tr->source_hdl=ud_stream->source_hdl;
926 tr->socket=(ud_stream->condat).sock;
927 tr->transaction_id=transaction_id;
928
929 /* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */
930 if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; }
931
932 /* For the first load pass we only create objects */
933 if(ud_stream->load_pass==1) {
934 Object_t *obj=tr->object;
935 /* still we need to fill tr->K (last.pkey) field */
936 g_slist_foreach(obj->attributes, ud_each_primary_key_select, tr);
937 tr->object_id=0;
938 }
939 else tr->object_id=get_object_id(tr);
940
941 /* Object cannot be retrieved */
942 if(tr->object_id==-1) { /* DB error*/
943 tr->succeeded=0;
944 tr->error |= ERROR_U_DBS;
945 ER_perror(FAC_UD, UD_SQL, "%s: Object cannot be retrieved", parse->object_name);
946 die;
947 transaction_free(tr);
948 object_free(obj);
949 }
950 /* save the name of person/role as we need it for referential */
951 /* integrity check when deleting the object against names. */
952 /* This is needed to support legacy references by name rather */
953 /* then by nic_hdl */
954 if((tr->class_type==C_PN) || (tr->class_type==C_RO)){
955 /* Save the value */
956 tr->save=g_strdup(parse->object_name);
957 /* attribute_free(attr, NULL); */
958 }
959
960 /* Process transaction. tr and obj are freed inside the process_* functions */
961
962 if(IS_UPDATE(ud_stream->ud_mode))
963 /* We are in update mode */
964 result=process_updates(ud_stream, tr, operation);
965 else
966 /* We are in NRTM mode */
967 result=process_nrtm(ud_stream, tr, operation);
968
969 return(result);
970
971 }
972
973
974 /************************************************************
975 * *
976 * int UD_process_stream(UD_stream_t *ud_stream) *
977 * *
978 * Processes the stream *
979 * *
980 * ud_stream - pointer to UD_stream_t structure *
981 * *
982 * Returns: *
983 * in update mode (!standalone)(1 object processed): *
984 * 1 - no error *
985 * <0- errors *
986 * *
987 * in NRTM & standalone modes *
988 * total number of object processed *
989 * *
990 ************************************************************/
991
992 int UD_process_stream(UD_stream_t *ud_stream)
/* [<][>][^][v][top][bottom][index][help] */
993 {
994 char line_buff[STR_XXL];
995 Object_t *obj = NULL;
996 SQ_connection_t *sql_connection;
997 int start_object;
998 int a_type;
999 struct _nrtm *nrtm;
1000 Log_t *log_ptr= &(ud_stream->log);
1001 ut_timer_t stime, ftime, sotime;
1002 float obj_second1, obj_second10, timediff;
1003 int result=0;
1004 int operation=0;
1005 int interrupt=0;
1006 int do_update;
1007 int default_ud_mode = ud_stream->ud_mode;
1008 Line_Type_t linetype;
1009 Transaction_t *tr;
1010 long transaction_id=0; /* transaction_id (to be supplied by DBupdate and stored in Database) */
1011 long serial_id;
1012 Obj_parse_t obj_parse; /* the structure used to parse a text object */
1013
1014
1015
1016
1017 nrtm=ud_stream->nrtm;
1018 start_object = 1;
1019 a_type=-1;
1020
1021
1022 /* Check connection to the database */
1023 if(SQ_ping(ud_stream->db_connection)) {
1024 ER_perror(FAC_UD, UD_SQL, "%s", SQ_error(ud_stream->db_connection));
1025 die;
1026 }
1027
1028 sql_connection=ud_stream->db_connection;
1029
1030 ud_parse_init(sql_connection, &obj_parse);
1031 /* Start timer for statistics */
1032 UT_timeget(&stime);
1033
1034 /* Main loop. Reading input stream line by line */
1035 /* Empty line signals to start processing an object, if we have it */
1036 while (SK_cd_gets(&ud_stream->condat, line_buff, sizeof(line_buff))>0) {
1037
1038
1039 switch (linetype=line_type(line_buff, &transaction_id)) {
1040 case LINE_ATTRIBUTE:
1041 /* parse the object line by line */
1042 obj = UD_parse_object(&obj_parse, line_buff);
1043
1044 break;
1045
1046 case LINE_COMMENT:
1047 break;
1048
1049 case LINE_EOF:
1050 break;
1051
1052 case LINE_ACK:
1053 tr = transaction_new(ud_stream->db_connection, 0);
1054 tr->transaction_id=transaction_id;
1055 TR_delete_record(tr);
1056 transaction_free(tr);
1057 break;
1058
1059
1060 case LINE_ADD:
1061 /* restore the default operation mode */
1062 operation=OP_ADD;
1063 ud_stream->ud_mode=default_ud_mode;
1064 break;
1065
1066 case LINE_OVERRIDE_ADD:
1067 /* for override - switch the dummy bit on */
1068 operation=OP_ADD;
1069 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1070 break;
1071
1072 case LINE_UPD:
1073 /* restore the default operation mode */
1074 operation=OP_UPD;
1075 ud_stream->ud_mode=default_ud_mode;
1076 break;
1077
1078 case LINE_OVERRIDE_UPD:
1079 /* for override - switch the dummy bit on */
1080 operation=OP_UPD;
1081 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1082 break;
1083
1084 case LINE_DEL:
1085 /* restore the default operation mode */
1086 operation=OP_DEL;
1087 ud_stream->ud_mode=default_ud_mode;
1088 break;
1089
1090 case LINE_OVERRIDE_DEL:
1091 /* for override - switch the dummy bit on */
1092 operation=OP_DEL;
1093 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1094 break;
1095
1096 case LINE_EMPTY:
1097 /* start processing the object */
1098 if ((obj=obj_parse.obj)) { /* if not just garbage*/
1099 /* normalize the object (reorder and split attributes */
1100 ud_normalize(&obj_parse);
1101 /* XXX */
1102 /* print_object(obj); */
1103
1104 /* start new transaction now */
1105 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: [%s] ", transaction_id, obj_parse.object_name);
1106
1107 /* fprintf(stderr, "transction # %ld\n", transaction_id); */
1108 result=process_transaction(ud_stream, &obj_parse, operation, transaction_id);
1109 /* process_transaction() frees tr and obj structures, */
1110 /* so make sure we'll not reference these objects in the future */
1111 operation=OP_NOOP;
1112 transaction_id=0;
1113 ud_stream->ud_mode=default_ud_mode;
1114 ud_parse_free(&obj_parse);
1115
1116 /* this is a good place for quick interrupt */
1117 do_update=CO_get_do_update();
1118 if (do_update) interrupt=0; else interrupt=1;
1119 } /* if this is a real object */
1120 /* initialize the parsing structure */
1121 ud_parse_init(sql_connection, &obj_parse);
1122
1123 break;
1124
1125 default:
1126 die;
1127 } /* switch */
1128
1129 /* Finish processing if interrupt has been set */
1130 if (interrupt) break;
1131 } /* Main loop of data stream processing : while */
1132
1133 /* Some postprocessing */
1134 if(IS_NRTM_CLNT(ud_stream->ud_mode)){
1135 /* We are in NRTM mode */
1136 /* Clean up */
1137 /* fclose(ud_stream->stream); */
1138 /* In NRTM mode there may be a saved object that is unprocessed */
1139 if(nrtm->tr){ /*saved backlog?*/
1140 /* restart the timer for statistics */
1141 UT_timeget(&sotime);
1142 object_process(nrtm->tr); /* delete the previous(saved) object*/
1143 /* result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name,
1144 "NRTM:DEL:While deleting previous(saved) object"); */
1145 /* create DEL serial record no matter what the result is */
1146 UD_lock_serial(nrtm->tr);
1147 serial_id = UD_create_serial(nrtm->tr);
1148 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
1149 UD_commit_serial(nrtm->tr);
1150 UD_unlock_serial(nrtm->tr);
1151 /* Mark TR as clean */
1152 TR_mark_clean(nrtm->tr);
1153 /* log the transaction */
1154 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
1155
1156 object_free(nrtm->tr->object);
1157 transaction_free(nrtm->tr); nrtm->tr=NULL;
1158 }
1159 }
1160
1161 /* That's all. Free GString */
1162 /* g_string_free(g_line_buff, TRUE);*/
1163
1164
1165 /* Calculate some statistics */
1166 /* ftime=time(NULL); */
1167 UT_timeget(&ftime);
1168 /* obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime);
1169 obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime); */
1170 timediff = UT_timediff(&stime, &ftime);
1171 obj_second1 = (float)(log_ptr->num_ok)/timediff;
1172 obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/timediff;
1173
1174 /* Print the report */
1175 if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) {
1176
1177 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s ******** report **********", UD_TAG);
1178 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects OK (%7.4f obj/s)", UD_TAG, log_ptr->num_ok, obj_second1);
1179 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects failed", UD_TAG, log_ptr->num_failed);
1180 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s average processing time %7.4f obj/s (%6.2f obj/min)", UD_TAG,
1181 obj_second10, obj_second10*60);
1182 result=log_ptr->num_ok+log_ptr->num_failed;
1183 }
1184 return(result);
1185
1186 } /* UD_process_stream */
1187