modules/ud/ud_process_stream.c
/* [<][>][^][v][top][bottom][index][help] */
FUNCTIONS
This source file includes following functions.
- ud_parse_init
- ud_parse_free
- line_continuation
- ud_replace_substring
- ud_split_attribute
- each_attribute_print
- print_object
- escape_apostrophes
- line_type
- UD_parse_object
- each_attribute_2_pass
- ud_normalize
- report_transaction
- process_nrtm
- process_updates
- process_transaction
- UD_process_stream
1 /***************************************
2 $Revision: 1.53 $
3
4 Functions to process data stream( file, network socket, etc.)
5
6 Status: NOT REVUED, NOT TESTED
7
8 Author(s): Chris Ottrey, Andrei Robachevsky
9
10 ******************/ /******************
11 Modification History:
12 andrei (17/01/2000) Created.
13 ******************/ /******************
14 Copyright (c) 2000 RIPE NCC
15
16 All Rights Reserved
17
18 Permission to use, copy, modify, and distribute this software and its
19 documentation for any purpose and without fee is hereby granted,
20 provided that the above copyright notice appear in all copies and that
21 both that copyright notice and this permission notice appear in
22 supporting documentation, and that the name of the author not be
23 used in advertising or publicity pertaining to distribution of the
24 software without specific, written prior permission.
25
26 THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
27 ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
28 AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
29 DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
30 AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
31 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 ***************************************/
33 #include <sys/types.h>
34 #include <sys/socket.h>
35 #include <netdb.h>
36 #include <arpa/inet.h>
37 #include <unistd.h>
38 #include <sys/stat.h>
39 #include <fcntl.h>
40 #include <string.h>
41 #include "constants.h"
42 #include "query_command.h"
43 #include "ud.h"
44 #include "ud_int.h"
45 #include "ud_tr.h"
46 #include "timediff.h"
47
48 typedef enum _Line_Type_t {
49 LINE_ATTRIBUTE,
50 LINE_COMMENT,
51 LINE_EMPTY,
52 LINE_EOF,
53 LINE_ADD,
54 LINE_UPD,
55 LINE_DEL,
56 LINE_OVERRIDE_ADD,
57 LINE_OVERRIDE_UPD,
58 LINE_OVERRIDE_DEL,
59 LINE_ACK
60 } Line_Type_t;
61
62 /* Maximum number of objects(serials) we can consume at a time */
63 #define SBUNCH 1000
64
65 static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason);
66 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
67 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
68 static int process_transaction(UD_stream_t *ud_stream, Obj_parse_t *parse, int operation, long transaction_id);
69
70 static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr);
71 static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse);
72 static void ud_parse_free(Obj_parse_t *parse);
73 static int line_continuation(char *line);
74 static GString *escape_apostrophes(GString *text);
75 static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff);
76 static int ud_replace_substring(GString *gstring, char *old, char *new);
77 static int ud_normalize(Obj_parse_t *parse);
78
79 /* Delimiters that separate list members, both RPS(,) and legacy( ) */
80 #define ATTR_DELIMITERS " ,\t"
81
82
83 static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse){
/* [<][>][^][v][top][bottom][index][help] */
84 bzero(parse, sizeof(Obj_parse_t));
85 parse->garbage = 0;
86 parse->sql_connection = sql_connection;
87 }
88
89 static void ud_parse_free(Obj_parse_t *parse){
/* [<][>][^][v][top][bottom][index][help] */
90 UT_free(parse->nic);
91 UT_free(parse->object_name);
92 }
93
94
95 /******************************************************************
96 * int line_continuation() *
97 * *
98 * Checks if the line starts with one of line continuation signs *
99 * Returns 1 if this is line continuation, 0 otherwise *
100 * *
101 * ****************************************************************/
102 static int line_continuation(char *line)
/* [<][>][^][v][top][bottom][index][help] */
103 {
104 switch(*line) {
105 case ' ':
106 case '\t':
107 case '+':
108 return(1); /* these indicate line continuation */
109 default: return(0);
110 }
111
112 }
113
114 /******************************************************************
115 * int ud_replace_substring() *
116 * *
117 * replace a subsring in a string *
118 * Returns -1 if no match was found *
119 * *
120 * ****************************************************************/
121 static int ud_replace_substring(GString *gstring, char *old, char *new)
/* [<][>][^][v][top][bottom][index][help] */
122 {
123 int i, i0=-1, j=0;
124
125 for (i=0; ((i < gstring->len) && (old[j] != '\0')); i++){
126 if(gstring->str[i]==old[j]){
127 if(i0==-1)i0=i;
128 j++;
129 } else {
130 i0=-1;
131 j=0;
132 }
133 }
134 /* if we haven't reached the end of the old substring */
135 /* or no match occured return -1 */
136 if((old[j] != '\0') || (i0==-1)) return (-1);
137 g_string_erase(gstring, i0, j);
138 g_string_insert(gstring, i0, new);
139 return(0);
140 }
141
142 /************************************************************
143 * *
144 * The function to splits attribute value into multiple *
145 * words and appends them as attr_type - attr_valu pairs *
146 * to the attr_list *
147 * *
148 * *
149 ************************************************************/
150 static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr)
/* [<][>][^][v][top][bottom][index][help] */
151 {
152 char *token;
153 char *split;
154 char *value, *n;
155 Attribute_t *attr_split;
156 GSList *the_list = attr_list;
157
158
159 /* check for line continuation (+) */
160 if (strncmp(attr->value, "+", 1) == 0) attr->value++;
161 /* check for end-of-line comments */
162 n = index(attr->value, '#');
163 /* if there is no comment check for trailing \n */
164 if(n == NULL) n = index(attr->value, '\n');
165 /* now copy the clean value into the attribute */
166 if(n == NULL) value = g_strdup(attr->value);
167 else value = g_strndup(attr->value, (n - attr->value));
168
169 token=value;
170 while((split=strsep(&token, ATTR_DELIMITERS))){
171 if (*split != '\0'){
172 attr_split = attribute_new1(attr->type, split);
173 if (attr_split) the_list = g_slist_append(the_list, attr_split);
174 }
175 }
176 free(value);
177 attribute_free(attr, NULL);
178
179 return(the_list);
180 }
181
182 /* XXX */
183 static void each_attribute_print(void *element_data, void *tr_ptr)
/* [<][>][^][v][top][bottom][index][help] */
184 {
185
186 Attribute_t *attr = (Attribute_t *)element_data;
187
188 fprintf(stderr, "[%d|%s]\n", attr->type, attr->value);
189
190 }
191
192 /* XXX */
193 static void print_object(Object_t *obj)
/* [<][>][^][v][top][bottom][index][help] */
194 {
195 g_slist_foreach(obj->attributes, each_attribute_print, NULL);
196 fprintf(stderr, ">>>>>\n%s\n", obj->object->str);
197 }
198
199
200 /******************************************************************
201 * GString *escape_apostrophes() *
202 * Escapes apostrophes in the text so they do not confuse printf *
203 * functions and don't corrupt SQL queries *
204 * *
205 * *****************************************************************/
206 static GString *escape_apostrophes(GString *text) {
/* [<][>][^][v][top][bottom][index][help] */
207 int i;
208 for (i=0; i < text->len; i++) {
209 if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
210 text = g_string_insert_c(text, i, '\\');
211 i++;
212 }
213 }
214 return(text);
215 } /* escape_apostrophes() */
216
217
218 /******************************************************************
219 * Line_Type_t line_type(e) *
220 * Determines the line type analysing the first letters *
221 * *
222 * ****************************************************************/
223 static Line_Type_t line_type(const char *line, long *transaction_id) {
/* [<][>][^][v][top][bottom][index][help] */
224
225 if (strncmp(line, "# EOF", 4) == 0) return(LINE_EOF);
226 if (strncmp(line, "#", 1) == 0) return(LINE_COMMENT);
227 if (strcmp(line, "\n") == 0) return(LINE_EMPTY);
228
229 if (strncmp(line, "ACK", 3) == 0) {
230 *transaction_id = atol(line+3);
231 return(LINE_ACK);
232 }
233 if (strncmp(line, "ADD_OVERRIDE", 12) == 0) {
234 *transaction_id = atol(line+12);
235 return(LINE_OVERRIDE_ADD);
236 }
237 if (strncmp(line, "UPD_OVERRIDE", 12) == 0) {
238 *transaction_id = atol(line+12);
239 return(LINE_OVERRIDE_UPD);
240 }
241 if (strncmp(line, "DEL_OVERRIDE", 12) == 0) {
242 *transaction_id = atol(line+12);
243 return(LINE_OVERRIDE_DEL);
244 }
245
246 if (strncmp(line, "ADD", 3) == 0) {
247 *transaction_id = atol(line+3);
248 return(LINE_ADD);
249 }
250 if (strncmp(line, "UPD", 3) == 0) {
251 *transaction_id = atol(line+3);
252 return(LINE_UPD);
253 }
254 if (strncmp(line, "DEL", 3) == 0) {
255 *transaction_id = atol(line+3);
256 return(LINE_DEL);
257 }
258
259 /* Otherwise this is an attribute */
260 return(LINE_ATTRIBUTE);
261
262 } /* line_type() */
263
264 /******************************************************************
265 * Object_t *UD_parse_object() *
266 * *
267 * Parses the object accepting line by line *
268 * Stores the object and list of attributes in Obj_parse_t struct *
269 * *
270 * ****************************************************************/
271 static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff)
/* [<][>][^][v][top][bottom][index][help] */
272 {
273 GString *g_line_buff;
274 Attribute_t *attr;
275
276 /* in case of garbage */
277 if(parse->garbage) return(NULL);
278
279 if(parse->obj==NULL) {
280 /* which means this is the start - create a new object*/
281 parse->obj = object_new(line_buff);
282 /* if this is not a class attribute - this is a garbage till the next blank line */
283 if(parse->obj==NULL) {
284 parse->garbage = 1;
285 return(NULL);
286 }
287 }
288
289 /* allocate buffer or the string */
290 if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){
291 ER_perror(FAC_UD, UD_MEM, "cannot allocate gstring");
292 die;
293 }
294 /* copy the input */
295 g_string_sprintf(g_line_buff, "%s", line_buff);
296 /* escape apostrophes in the input line */
297 g_line_buff=escape_apostrophes(g_line_buff);
298
299 attr = attribute_new(g_line_buff->str);
300 if (attr){
301 /* save this attribute for future line continuations */
302 /* skip attributes that are not stored in SQL database */
303 if(DF_get_update_query_type(attr->type)==UD_NULL_){
304 parse->current_attr = NULL;
305 attribute_free(attr, NULL);
306 }
307 else {
308 parse->current_attr = attr;
309 if((attr->type == A_MB) || (attr->type == A_NH)) {
310 /* insert it at the beginning */
311 /* mnt-by should go before member-of to allow correct membership autorization (still done in RIPupd) */
312 /* nic-hdl should go before any admin-c, tech-c to prevent errrors in self referencing role objects */
313 parse->obj->attributes = g_slist_insert(parse->obj->attributes, parse->current_attr, 1);
314 } else {
315 /* append it to the attribute list */
316 parse->obj->attributes = g_slist_append(parse->obj->attributes, parse->current_attr);
317 }
318 }
319 }
320 /* if this is not a valid attribute, then this may be unknown attr or line continuation */
321 else
322 if(line_continuation(g_line_buff->str))
323 {
324 /* if parse->attr == NULL, then this attr is not stored in SQL database - skip it */
325 if(parse->current_attr){
326 /* this is continuation of a previous attribute */
327 char *line_value, *new_value;
328 char *n;
329 /* normalize and append to current_attr->value */
330 /* check for end-of-line comments */
331 n = index(g_line_buff->str, '#');
332 /* if there is no comment check for trailing \n */
333 if(n == NULL) n = index(g_line_buff->str, '\n');
334 /* now copy the clean value into the attribute */
335 if(n == NULL) line_value = g_strdup(g_line_buff->str);
336 else line_value = g_strndup(g_line_buff->str, (n - g_line_buff->str));
337 /* replace line continuation character with ' ' */
338 *line_value = ' ';
339 new_value = g_strconcat(parse->current_attr->value, line_value, NULL);
340 parse->current_attr = attribute_upd(parse->current_attr, parse->current_attr->type, new_value);
341 UT_free(new_value);
342 UT_free(line_value);
343 }
344 }
345 /* otherwise, this is unknown attribute, so we need to break the continuation */
346 else parse->current_attr=NULL;
347 /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */
348 g_string_sprintfa(parse->obj->object, "%s", g_line_buff->str);
349 g_string_free(g_line_buff, TRUE);
350 return(parse->obj);
351 }
352
353 /******************************************************************
354 * int each_attribute_2_pass() *
355 * *
356 * Assigns nic-handles in case of AUTO *
357 * Splits attributes *
358 * * *
359 ******************************************************************/
360 void each_attribute_2_pass(void *element_data, void *ptr)
/* [<][>][^][v][top][bottom][index][help] */
361 {
362 Attribute_t *attr = (Attribute_t *)element_data;
363 Obj_parse_t *parse = (Obj_parse_t *)ptr;
364
365
366 /* Strip the white space */
367 g_strstrip(attr->value);
368
369 switch(attr->type){
370 case A_NH: /* nic-hdl */
371 /* Parse the string into nh structure */
372 /* In case of an AUTO NIC handle check the ID in the database */
373 /* Possible errors leave to core processing */
374 if(NH_parse(attr->value, &parse->nh_ptr) == 0) {
375 /* this is an AUTO nic handle */
376 /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsing nic handle: [%s]", UD_TAG, attr->value);*/
377 /* Check if we can allocate it */
378 if(NH_check(parse->nh_ptr, parse->sql_connection)>0){
379 /* Convert nh to the database format */
380 parse->nic = NH_convert(parse->nh_ptr);
381 /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsed and converted nic handle: [%s]", UD_TAG, nic); */
382 /* replace the auto nic with a real one */
383 if(ud_replace_substring(parse->obj->object, attr->value, parse->nic)==-1) {
384 ER_perror(FAC_UD, UD_BUG, "cannot replace auto nic handle");
385 die;
386 }
387 /* Update the attribute */
388 attr = attribute_upd(attr, attr->type, parse->nic);
389
390 }
391 }
392 parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
393 break;
394 case A_PN: /* person */
395 case A_RO: /* role */
396 case A_MR: /* mbrs-by-ref */
397 case A_MB: /* mnt-by */
398 case A_ML: /* mnt-lower */
399 case A_CT: /* cross-mnt */
400 case A_MO: /* member-of */
401 case A_SD: /* sub-dom */
402 case A_RZ: /* rev-srv */
403 case A_NS: /* nserver */
404 parse->new_attr_list = ud_split_attribute(parse->new_attr_list, attr);
405 break;
406 default:
407 parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
408 break;
409 }
410
411 }
412
413
414 /******************************************************************
415 * int ud_normalize() *
416 * *
417 * function accepts Obj_parse_t structure with attribute list *
418 * *
419 * It makes another pass to normalize object *
420 * Normalization includes: *
421 * *
422 * 1) reorder attributes *
423 * . mnt-by should go before member-of to allow correct *
424 * membership autorization (still done in RIPupd) *
425 * . nic-hdl should go before any admin-c, tech-c to prevent *
426 * errrors in self referencing role objects *
427 * 2) in case of pn/ro check nic handle and replace it in *
428 * the obj->object if needed (AUTO) *
429 * 3) split some attributes that allow lists *
430 ******************************************************************/
431 static int ud_normalize(Obj_parse_t *parse)
/* [<][>][^][v][top][bottom][index][help] */
432 {
433 GSList *old_attr_list;
434 GSList *first_element;
435 Attribute_t *class_attr;
436
437 old_attr_list = parse->obj->attributes;
438 /* get class attribute - the first one */
439 first_element = g_slist_nth(old_attr_list, 0);
440 class_attr = (Attribute_t *)first_element->data;
441 /* save object name for reporting and checking ref integrity for names (legacy) */
442 parse->object_name = g_strdup(class_attr->value);
443 /* reorder the attributes mnt-by and nic-hdl come first */
444 /*g_slist_foreach(old_attr_list, each_attribute_1_pass, parse);
445 g_slist_free(old_attr_list);
446 old_attr_list = parse->new_attr_list; */
447 parse->new_attr_list = NULL;
448 /* assign auto nic-hdl and slit attributes */
449 g_slist_foreach(old_attr_list, each_attribute_2_pass, parse);
450 g_slist_free(old_attr_list);
451 parse->obj->attributes = parse->new_attr_list;
452
453 return(1);
454 }
455
456
457 /******************************************************************
458 * report_transaction() *
459 * *
460 * Prints error report to the log *
461 * *
462 * reason - additional message that will be included *
463 * *
464 * *****************************************************************/
465 static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason)
/* [<][>][^][v][top][bottom][index][help] */
466 {
467 int result=0;
468 ut_timer_t fotime;
469 float timediff;
470 const char *class_name = DF_class_type2name(tr->class_type);
471 char *primary_key = tr->K->str;
472
473
474 /* calculate statistics */
475 UT_timeget(&fotime);
476 timediff = UT_timediff(psotime, &fotime);
477
478 if(tr->succeeded==0) {
479 result=tr->error;
480 log->num_failed++;
481 ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] %.2fs FAILED [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
482 /* ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] object: FAILED [%s][%s](%d/%d)", transaction_id, , reason, log->num_failed, (log->num_failed)+(log->num_ok)); */
483 if(result & ERROR_U_OBJ) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: referential integrity error", transaction_id);
484 if(result & ERROR_U_AUT) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: authentication error", transaction_id);
485 if(result & ERROR_U_BADOP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: unsupported operation", transaction_id);
486 if(result & ERROR_U_COP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: conflicting operation", transaction_id);
487 if(result & ERROR_U_NSUP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: this type is not supported", transaction_id);
488 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
489 result=1; /* # of failures */
490 }
491 else {
492 result=0;
493 log->num_ok++;
494 /* ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: OK [%s](%d/%d)", transaction_id, obj_name, log->num_ok, (log->num_failed)+(log->num_ok)); */
495 ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] %.2fs OK [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
496 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
497 }
498
499 return(result);
500 }/* report_transaction() */
501
502
503
504 /************************************************************
505 * process_nrtm() *
506 * *
507 * Process object in NRTM client mode *
508 * *
509 * nrtm - pointer to _nrtm structure *
510 * log - pointer to Log_t structure *
511 * object_name - name of the object *
512 * operation - operation code (OP_ADD/OP_DEL) *
513 * *
514 * Returns: *
515 * 1 - okay *
516 * <0 - error *
517 * *
518 ************************************************************/
519
520 static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
/* [<][>][^][v][top][bottom][index][help] */
521 {
522 int result=0;
523 int dummy=0;
524 struct _nrtm *nrtm = ud_stream->nrtm;
525 long serial_id;
526 Log_t *log_ptr= &(ud_stream->log);
527 ut_timer_t sotime;
528 int ta_upd_nhr;
529
530 /* Start timer for statistics */
531 UT_timeget(&sotime);
532
533 /* We allow NRTM updates for some inconsistent objects */
534 /* One of the examples is reference by name which looks like nic-handle */
535 /* For this purpose we allow dummy creation when updating an object */
536 /* We also check for dummy allowance when deleting an object */
537 /* this is done to allow deletion of person objects referenced by name */
538
539 tr->mode|=B_DUMMY;
540 if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
541
542 switch (operation) {
543
544 case OP_ADD:
545 if(nrtm->tr){ /* DEL ADD => saved*/
546 if(tr->object_id==0) {
547 /* object does not exist in the DB */
548 /* delete the previous(saved) object*/
549 object_process(nrtm->tr);
550 /* create DEL serial */
551 UD_lock_serial(nrtm->tr);
552 serial_id = UD_create_serial(nrtm->tr);
553 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
554 UD_commit_serial(nrtm->tr);
555 UD_unlock_serial(nrtm->tr);
556 /* Mark TR as clean */
557 TR_mark_clean(nrtm->tr);
558 /* log the transaction */
559 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
560
561 object_free(nrtm->tr->object);
562 transaction_free(nrtm->tr); nrtm->tr=NULL;
563
564 /* Create an object and update NHR */
565 tr->action=(TA_CREATE | ta_upd_nhr);
566 /* restart the timer for statistics */
567 UT_timeget(&sotime);
568 object_process(tr); /* create a new one*/
569 /* create ADD serial */
570 UD_lock_serial(tr);
571 serial_id = UD_create_serial(tr);
572 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
573 UD_commit_serial(tr);
574 UD_unlock_serial(tr);
575 /* Mark TR as clean */
576 TR_mark_clean(tr);
577 /* log the transaction */
578 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
579 }
580 else {
581 /* object already exists in the DB - update or dummy replacement*/
582 /*compare the two, may be we may collapse operations*/
583 if(tr->object_id==nrtm->tr->object_id) {
584 /* DEL-ADD ->> UPDATE */
585 object_free(nrtm->tr->object);
586 transaction_free(nrtm->tr); nrtm->tr=NULL;
587 tr->action=TA_UPD_CLLPS;
588 object_process(tr);
589 /* create DEL+ADD serial records */
590 UD_lock_serial(tr);
591 tr->action=TA_DELETE; serial_id = UD_create_serial(tr);
592 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL(UPD)");
593
594 /* restart the timer for statistics */
595 UT_timeget(&sotime);
596 tr->sequence_id++;
597 tr->action=TA_CREATE; serial_id = UD_create_serial(tr);
598 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
599 UD_commit_serial(tr);
600 UD_unlock_serial(tr);
601 /* Mark TR as clean */
602 TR_mark_clean(tr);
603 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD(UPD)");
604 }
605 else { /* this should be a dummy object in the database(that we are going to replace with the real one */
606 /* or an interleaved operation*/
607 object_process(nrtm->tr); /* delete the previous(saved) object*/
608 /* create a DEL serial record */
609 UD_lock_serial(nrtm->tr);
610 serial_id = UD_create_serial(nrtm->tr);
611 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
612 UD_commit_serial(nrtm->tr);
613 UD_unlock_serial(nrtm->tr);
614 /* Mark TR as clean */
615 TR_mark_clean(nrtm->tr);
616 /* result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");*/
617 /* log the transaction */
618 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
619
620
621 object_free(nrtm->tr->object);
622 transaction_free(nrtm->tr); nrtm->tr=NULL;
623
624 /* restart the timer for statistics */
625 UT_timeget(&sotime);
626
627 tr->action=TA_UPDATE;
628 /* check if we are replacing a dummy object */
629 dummy=isdummy(tr);
630 /* If we are replacing dummy with a real object update NHR */
631 if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
632 /* fprintf(stderr,"UPDATE next(dummy)\n"); */
633 object_process(tr); /* create a new one*/
634 /* result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new"); */
635 /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
636 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
637 /* create ADD serial record */
638 UD_lock_serial(tr);
639 serial_id = UD_create_serial(tr);
640 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
641 UD_commit_serial(tr);
642 UD_unlock_serial(tr);
643 /* Mark TR as clean */
644 TR_mark_clean(tr);
645 /* log the transaction */
646 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
647
648 }
649 }
650 }
651 else { /* ADD ADD =>brand new object*/
652 if(tr->object_id==0) {
653 /* fprintf(stderr,"CREATE new\n");*/
654 /* Create an object and update NHR */
655 tr->action=(TA_CREATE | ta_upd_nhr);
656 object_process(tr);
657 /* create ADD serial */
658 UD_lock_serial(tr);
659 serial_id = UD_create_serial(tr);
660 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
661 UD_commit_serial(tr);
662 UD_unlock_serial(tr);
663
664 /* Mark TR as clean */
665 TR_mark_clean(tr);
666 /* log the transaction */
667 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
668 }
669 else { /* object already exists in the database */
670 /* this may happen because of dummies*/
671 /* or with some implementations of mirroring protocol that have atomic update */
672 /* instead of add + del */
673 tr->action=TA_UPDATE;
674 dummy=isdummy(tr);
675 /* If we are replacing dummy with a real object update NHR */
676 if(dummy==1) tr->action = ( ta_upd_nhr| TA_UPD_DUMMY);
677 object_process(tr);
678 /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
679 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
680 /* create ADD serial record */
681 UD_lock_serial(tr);
682 serial_id = UD_create_serial(tr);
683 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
684 UD_commit_serial(tr);
685 UD_unlock_serial(tr);
686 /* Mark TR as clean */
687 TR_mark_clean(tr);
688 /* log the transaction */
689 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
690
691 }
692 }
693 break;
694
695 case OP_DEL:
696 if(nrtm->tr){ /*DEL DEL =>saved */
697 /* check this is not a deletion of the same object twise */
698 /* this should not happen but we cannot trust foreign sources */
699 /* in such case process saved transaction but fail the current one */
700 if(nrtm->tr->object_id == tr->object_id) tr->object_id=0;
701
702 object_process(nrtm->tr); /* delete the previous(saved) object*/
703 /* create DEL serial record */
704 UD_lock_serial(nrtm->tr);
705 serial_id = UD_create_serial(nrtm->tr);
706 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
707 UD_commit_serial(nrtm->tr);
708 UD_unlock_serial(nrtm->tr);
709 /* Mark TR as clean */
710 TR_mark_clean(nrtm->tr);
711 /* log the transaction */
712 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
713
714 object_free(nrtm->tr->object);
715 transaction_free(nrtm->tr); nrtm->tr=NULL;
716 }
717 /* save the real object (not a dummy one ) */
718 if(tr->object_id>0 && !isdummy(tr)){
719 /* save the object*/
720 tr->action=(TA_DELETE | ta_upd_nhr);
721 nrtm->tr=tr;
722 return(0);
723 }
724 else { /* this is an error - Trying to DEL non-existing object*/
725 tr->succeeded=0; tr->error|=ERROR_U_COP;
726 tr->action=(TA_DELETE | ta_upd_nhr);
727 /* create and initialize TR record for crash recovery */
728 TR_create_record(tr);
729 /* create DEL serial record anyway */
730 UD_lock_serial(tr);
731 serial_id = UD_create_serial(tr);
732 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
733 UD_commit_serial(tr);
734 UD_unlock_serial(tr);
735 /* Mark TR as clean */
736 TR_mark_clean(tr);
737 /* log the transaction */
738 result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL: non-existing object");
739
740 }
741 break;
742
743 default:
744 tr->succeeded=0; tr->error |=ERROR_U_BADOP;
745 break;
746 }
747
748 /* Free resources */
749 object_free(tr->object);
750 transaction_free(tr);
751
752 return(result);
753 } /* process_nrtm() */
754
755
756
757 /************************************************************
758 * process_updates() *
759 * *
760 * Process object in update mode *
761 * *
762 * ud_stream - pointer to UD_stream structure *
763 * object_name - name of the object *
764 * operation - operation code (OP_ADD/OP_DEL) *
765 * *
766 * Note: *
767 * Frees tr and tr->obj on exit *
768 * *
769 * Returns: *
770 * 0 - okay *
771 * <0- number of failed objects *
772 * *
773 ************************************************************/
774
775 static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
/* [<][>][^][v][top][bottom][index][help] */
776 {
777 int result=0;
778 Log_t *log_ptr= &(ud_stream->log);
779 int dummy=0;
780 ut_timer_t sotime;
781 int ta_upd_nhr;
782 char *reason;
783
784 /* Start timer for statistics */
785 UT_timeget(&sotime);
786 if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
787
788 switch(operation) {
789 /* Compare operations and report an error if they do not match */
790 case OP_ADD:
791 if(tr->object_id!=0) { /* trying to create, but object exists */
792 tr->succeeded=0; tr->error|=ERROR_U_COP;
793 reason="U:ADD:object already exists";
794 UD_ack(tr); /* Send a NACK */
795 } else {
796 /* Action: create the object and update NHR */
797 tr->action=(TA_CREATE | ta_upd_nhr);
798 reason="U:ADD";
799 object_process(tr);
800 }
801 break;
802 case OP_UPD:
803 if(tr->object_id==0) { /* trying to update non-existing object*/
804 tr->succeeded=0; tr->error|=ERROR_U_COP;
805 reason="U:UPD:non-existing object";
806 UD_ack(tr); /* Send a NACK */
807 } else {
808 tr->action=TA_UPDATE;
809 reason="U:UPD";
810 dummy=isdummy(tr);
811 /* If we are replacing dummy with a real object update NHR */
812 if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
813 object_process(tr);
814 }
815 break;
816
817 case OP_DEL:
818 if(tr->object_id==0) { /* trying t delete non-existing object*/
819 tr->succeeded=0; tr->error|=ERROR_U_COP;
820 reason="U:DEL:non-existing object";
821 UD_ack(tr);
822 } else {
823 tr->action=(TA_DELETE | ta_upd_nhr);
824 reason="U:DEL";
825 object_process(tr);
826 }
827 break;
828
829 default:
830 /* bad operation for this mode if not standalone */
831 if(IS_STANDALONE(tr->mode)) {
832 if(tr->object_id==0){
833 tr->action=(TA_CREATE | ta_upd_nhr);
834 reason="U:ADD";
835 }
836 else {
837 tr->action=TA_UPDATE;
838 reason="U:UPD";
839 }
840 object_process(tr);
841 }
842 else {
843 tr->succeeded=0;
844 tr->error|=ERROR_U_BADOP;
845 reason="U:bad operation";
846 UD_ack(tr); /* Send a NACK */
847 }
848 break;
849 }
850 /* If not in standalone mode create serial and copy error transcript */
851 if(!IS_STANDALONE(tr->mode)) {
852 if(tr->succeeded){
853 /* we don't want to generate DEL serial for dummy replacement*/
854 if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
855 UD_lock_serial(tr);
856 UD_create_serial(tr);
857 CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
858 UD_commit_serial(tr);
859 UD_unlock_serial(tr);
860 /* Mark the TR as clean */
861 TR_mark_clean(tr);
862 }
863 }
864
865 /* Make a report. U stands for update stream. No reason */
866 result=report_transaction(tr, tr->transaction_id, log_ptr, &sotime, reason);
867
868 /* Free resources */
869 object_free(tr->object);
870 transaction_free(tr);
871
872 return(result);
873
874 } /* process_updates() */
875
876
877 /************************************************************
878 * *
879 * int process_transaction() *
880 * *
881 * Processes the transaction *
882 * *
883 * ud_stream - pointer to UD_stream_t structure *
884 * *
885 * Returns: *
886 * 0 - no error *
887 * <0- number of failed objects *
888 * *
889 ************************************************************/
890
891 /* It frees the obj */
892
893 static int process_transaction(UD_stream_t *ud_stream,
/* [<][>][^][v][top][bottom][index][help] */
894 Obj_parse_t *parse,
895 int operation,
896 long transaction_id)
897 {
898 Transaction_t *tr = NULL;
899 Object_t *obj = parse->obj;
900 int result;
901
902 /* check if the requested transaction has already been processed */
903 /* this may happen in case of crash. If so, just send an ack and return */
904 if(TR_check(ud_stream->db_connection, transaction_id, (ud_stream->condat).sock))return(1);
905
906 /* start new transaction now */
907 tr = transaction_new(ud_stream->db_connection, obj->type);
908
909 /* Return with error if transaction cannot be created */
910 if (tr == NULL) die;
911
912 tr->mode=ud_stream->ud_mode;
913 tr->load_pass=ud_stream->load_pass;
914 tr->object=obj;
915 tr->nh=parse->nh_ptr;
916 tr->source_hdl=ud_stream->source_hdl;
917 tr->socket=(ud_stream->condat).sock;
918 tr->transaction_id=transaction_id;
919
920 /* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */
921 if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; }
922
923 /* For the first load pass we only create objects */
924 if(ud_stream->load_pass==1) {
925 Object_t *obj=tr->object;
926 /* still we need to fill tr->K (last.pkey) field */
927 g_slist_foreach(obj->attributes, ud_each_primary_key_select, tr);
928 tr->object_id=0;
929 }
930 else tr->object_id=get_object_id(tr);
931
932 /* Object cannot be retrieved */
933 if(tr->object_id==-1) { /* DB error*/
934 tr->succeeded=0;
935 tr->error |= ERROR_U_DBS;
936 ER_perror(FAC_UD, UD_SQL, "%s: Object cannot be retrieved", parse->object_name);
937 die;
938 transaction_free(tr);
939 object_free(obj);
940 }
941 /* save the name of person/role as we need it for referential */
942 /* integrity check when deleting the object against names. */
943 /* This is needed to support legacy references by name rather */
944 /* then by nic_hdl */
945 if((tr->class_type==C_PN) || (tr->class_type==C_RO)){
946 /* Save the value */
947 tr->save=g_strdup(parse->object_name);
948 /* attribute_free(attr, NULL); */
949 }
950
951 /* Process transaction. tr and obj are freed inside the process_* functions */
952
953 if(IS_UPDATE(ud_stream->ud_mode))
954 /* We are in update mode */
955 result=process_updates(ud_stream, tr, operation);
956 else
957 /* We are in NRTM mode */
958 result=process_nrtm(ud_stream, tr, operation);
959
960 return(result);
961
962 }
963
964
965 /************************************************************
966 * *
967 * int UD_process_stream(UD_stream_t *ud_stream) *
968 * *
969 * Processes the stream *
970 * *
971 * ud_stream - pointer to UD_stream_t structure *
972 * *
973 * Returns: *
974 * in update mode (!standalone)(1 object processed): *
975 * 1 - no error *
976 * <0- errors *
977 * *
978 * in NRTM & standalone modes *
979 * total number of object processed *
980 * *
981 ************************************************************/
982
983 int UD_process_stream(UD_stream_t *ud_stream)
/* [<][>][^][v][top][bottom][index][help] */
984 {
985 char line_buff[STR_XXL];
986 Object_t *obj = NULL;
987 SQ_connection_t *sql_connection;
988 int start_object;
989 int a_type;
990 struct _nrtm *nrtm;
991 Log_t *log_ptr= &(ud_stream->log);
992 ut_timer_t stime, ftime, sotime;
993 float obj_second1, obj_second10, timediff;
994 int result;
995 int operation=0;
996 int interrupt=0;
997 int do_update;
998 int default_ud_mode = ud_stream->ud_mode;
999 Line_Type_t linetype;
1000 Transaction_t *tr;
1001 long transaction_id=0; /* transaction_id (to be supplied by DBupdate and stored in Database) */
1002 long serial_id;
1003 Obj_parse_t obj_parse; /* the structure used to parse a text object */
1004
1005
1006
1007
1008 nrtm=ud_stream->nrtm;
1009 start_object = 1;
1010 a_type=-1;
1011
1012
1013 /* Check connection to the database */
1014 if(SQ_ping(ud_stream->db_connection)) {
1015 ER_perror(FAC_UD, UD_SQL, "%s", SQ_error(ud_stream->db_connection));
1016 die;
1017 }
1018
1019 sql_connection=ud_stream->db_connection;
1020
1021 ud_parse_init(sql_connection, &obj_parse);
1022 /* Start timer for statistics */
1023 UT_timeget(&stime);
1024
1025 /* Main loop. Reading input stream line by line */
1026 /* Empty line signals to start processing an object, if we have it */
1027 while (SK_cd_gets(&ud_stream->condat, line_buff, sizeof(line_buff))>0) {
1028
1029
1030 switch (linetype=line_type(line_buff, &transaction_id)) {
1031 case LINE_ATTRIBUTE:
1032 /* parse the object line by line */
1033 obj = UD_parse_object(&obj_parse, line_buff);
1034
1035 break;
1036
1037 case LINE_COMMENT:
1038 break;
1039
1040 case LINE_EOF:
1041 break;
1042
1043 case LINE_ACK:
1044 tr = transaction_new(ud_stream->db_connection, 0);
1045 tr->transaction_id=transaction_id;
1046 TR_delete_record(tr);
1047 transaction_free(tr);
1048 break;
1049
1050
1051 case LINE_ADD:
1052 /* restore the default operation mode */
1053 operation=OP_ADD;
1054 ud_stream->ud_mode=default_ud_mode;
1055 break;
1056
1057 case LINE_OVERRIDE_ADD:
1058 /* for override - switch the dummy bit on */
1059 operation=OP_ADD;
1060 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1061 break;
1062
1063 case LINE_UPD:
1064 /* restore the default operation mode */
1065 operation=OP_UPD;
1066 ud_stream->ud_mode=default_ud_mode;
1067 break;
1068
1069 case LINE_OVERRIDE_UPD:
1070 /* for override - switch the dummy bit on */
1071 operation=OP_UPD;
1072 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1073 break;
1074
1075 case LINE_DEL:
1076 /* restore the default operation mode */
1077 operation=OP_DEL;
1078 ud_stream->ud_mode=default_ud_mode;
1079 break;
1080
1081 case LINE_OVERRIDE_DEL:
1082 /* for override - switch the dummy bit on */
1083 operation=OP_DEL;
1084 ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1085 break;
1086
1087 case LINE_EMPTY:
1088 /* start processing the object */
1089 if ((obj=obj_parse.obj)) { /* if not just garbage*/
1090 /* normalize the object (reorder and split attributes */
1091 ud_normalize(&obj_parse);
1092 /* XXX */
1093 /* print_object(obj); */
1094
1095 /* start new transaction now */
1096 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: [%s] ", transaction_id, obj_parse.object_name);
1097
1098 /* fprintf(stderr, "transction # %ld\n", transaction_id); */
1099 result=process_transaction(ud_stream, &obj_parse, operation, transaction_id);
1100 /* process_transaction() frees tr and obj structures, */
1101 /* so make sure we'll not reference these objects in the future */
1102 operation=OP_NOOP;
1103 transaction_id=0;
1104 ud_stream->ud_mode=default_ud_mode;
1105 ud_parse_free(&obj_parse);
1106
1107 /* this is a good place for quick interrupt */
1108 do_update=CO_get_do_update();
1109 if (do_update) interrupt=0; else interrupt=1;
1110 } /* if this is a real object */
1111 /* initialize the parsing structure */
1112 ud_parse_init(sql_connection, &obj_parse);
1113
1114 break;
1115
1116 default:
1117 die;
1118 } /* switch */
1119
1120 /* Finish processing if interrupt has been set */
1121 if (interrupt) break;
1122 } /* Main loop of data stream processing : while */
1123
1124 /* Some postprocessing */
1125 if(IS_NRTM_CLNT(ud_stream->ud_mode)){
1126 /* We are in NRTM mode */
1127 /* Clean up */
1128 /* fclose(ud_stream->stream); */
1129 /* In NRTM mode there may be a saved object that is unprocessed */
1130 if(nrtm->tr){ /*saved backlog?*/
1131 /* restart the timer for statistics */
1132 UT_timeget(&sotime);
1133 object_process(nrtm->tr); /* delete the previous(saved) object*/
1134 /* result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name,
1135 "NRTM:DEL:While deleting previous(saved) object"); */
1136 /* create DEL serial record no matter what the result is */
1137 UD_lock_serial(nrtm->tr);
1138 serial_id = UD_create_serial(nrtm->tr);
1139 CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
1140 UD_commit_serial(nrtm->tr);
1141 UD_unlock_serial(nrtm->tr);
1142 /* Mark TR as clean */
1143 TR_mark_clean(nrtm->tr);
1144 /* log the transaction */
1145 result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
1146
1147 object_free(nrtm->tr->object);
1148 transaction_free(nrtm->tr); nrtm->tr=NULL;
1149 }
1150 }
1151
1152 /* That's all. Free GString */
1153 /* g_string_free(g_line_buff, TRUE);*/
1154
1155
1156 /* Calculate some statistics */
1157 /* ftime=time(NULL); */
1158 UT_timeget(&ftime);
1159 /* obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime);
1160 obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime); */
1161 timediff = UT_timediff(&stime, &ftime);
1162 obj_second1 = (float)(log_ptr->num_ok)/timediff;
1163 obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/timediff;
1164
1165 /* Print the report */
1166 if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) {
1167
1168 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s ******** report **********", UD_TAG);
1169 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects OK (%7.4f obj/s)", UD_TAG, log_ptr->num_ok, obj_second1);
1170 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects failed", UD_TAG, log_ptr->num_failed);
1171 ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s average processing time %7.4f obj/s (%6.2f obj/min)", UD_TAG,
1172 obj_second10, obj_second10*60);
1173 result=log_ptr->num_ok+log_ptr->num_failed;
1174 }
1175 return(result);
1176
1177 } /* UD_process_stream */
1178