1 | /*************************************** 2 | $Revision: 1.50 $ 3 | 4 | Functions to process data stream( file, network socket, etc.) 5 | 6 | Status: NOT REVUED, NOT TESTED 7 | 8 | Author(s): Chris Ottrey, Andrei Robachevsky 9 | 10 | ******************/ /****************** 11 | Modification History: 12 | andrei (17/01/2000) Created. 13 | ******************/ /****************** 14 | Copyright (c) 2000 RIPE NCC 15 | 16 | All Rights Reserved 17 | 18 | Permission to use, copy, modify, and distribute this software and its 19 | documentation for any purpose and without fee is hereby granted, 20 | provided that the above copyright notice appear in all copies and that 21 | both that copyright notice and this permission notice appear in 22 | supporting documentation, and that the name of the author not be 23 | used in advertising or publicity pertaining to distribution of the 24 | software without specific, written prior permission. 25 | 26 | THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING 27 | ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL 28 | AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY 29 | DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN 30 | AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 31 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 32 | ***************************************/ 33 | #include <sys/types.h> 34 | #include <sys/socket.h> 35 | #include <netdb.h> 36 | #include <arpa/inet.h> 37 | #include <unistd.h> 38 | #include <sys/stat.h> 39 | #include <fcntl.h> 40 | #include <string.h> 41 | #include "constants.h" 42 | #include "query_command.h" 43 | #include "ud.h" 44 | #include "ud_int.h" 45 | #include "ud_tr.h" 46 | #include "timediff.h" 47 | 48 | typedef enum _Line_Type_t { 49 | LINE_ATTRIBUTE, 50 | LINE_COMMENT, 51 | LINE_EMPTY, 52 | LINE_EOF, 53 | LINE_ADD, 54 | LINE_UPD, 55 | LINE_DEL, 56 | LINE_OVERRIDE_ADD, 57 | LINE_OVERRIDE_UPD, 58 | LINE_OVERRIDE_DEL, 59 | LINE_ACK 60 | } Line_Type_t; 61 | 62 | /* Maximum number of objects(serials) we can consume at a time */ 63 | #define SBUNCH 1000 64 | 65 | static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason); 66 | static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation); 67 | static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation); 68 | static int process_transaction(UD_stream_t *ud_stream, Obj_parse_t *parse, int operation, long transaction_id); 69 | 70 | static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr); 71 | static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse); 72 | static void ud_parse_free(Obj_parse_t *parse); 73 | static int line_continuation(char *line); 74 | static GString *escape_apostrophes(GString *text); 75 | static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff); 76 | static int ud_replace_substring(GString *gstring, char *old, char *new); 77 | static int ud_normalize(Obj_parse_t *parse); 78 | 79 | /* Delimiters that separate list members, both RPS(,) and legacy( ) */ 80 | #define ATTR_DELIMITERS " ," 81 | 82 | 83 | static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse){ 84 | bzero(parse, sizeof(Obj_parse_t)); 85 | parse->garbage = 0; 86 | parse->sql_connection = sql_connection; 87 | } 88 | 89 | static void ud_parse_free(Obj_parse_t *parse){ 90 | UT_free(parse->nic); 91 | UT_free(parse->object_name); 92 | } 93 | 94 | 95 | /****************************************************************** 96 | * int line_continuation() * 97 | * * 98 | * Checks if the line starts with one of line continuation signs * 99 | * Returns 1 if this is line continuation, 0 otherwise * 100 | * * 101 | * ****************************************************************/ 102 | static int line_continuation(char *line) 103 | { 104 | switch(*line) { 105 | case ' ': 106 | case '\t': 107 | case '+': 108 | return(1); /* these indicate line continuation */ 109 | default: return(0); 110 | } 111 | 112 | } 113 | 114 | /****************************************************************** 115 | * int ud_replace_substring() * 116 | * * 117 | * replace a subsring in a string * 118 | * Returns -1 if no match was found * 119 | * * 120 | * ****************************************************************/ 121 | static int ud_replace_substring(GString *gstring, char *old, char *new) 122 | { 123 | int i, i0=-1, j=0; 124 | 125 | for (i=0; ((i < gstring->len) && (old[j] != '\0')); i++){ 126 | if(gstring->str[i]==old[j]){ 127 | if(i0==-1)i0=i; 128 | j++; 129 | } else { 130 | i0=-1; 131 | j=0; 132 | } 133 | } 134 | /* if we haven't reached the end of the old substring */ 135 | /* or no match occured return -1 */ 136 | if((old[j] != '\0') || (i0==-1)) return (-1); 137 | g_string_erase(gstring, i0, j); 138 | g_string_insert(gstring, i0, new); 139 | return(0); 140 | } 141 | 142 | /************************************************************ 143 | * * 144 | * The function to splits attribute value into multiple * 145 | * words and appends them as attr_type - attr_valu pairs * 146 | * to the attr_list * 147 | * * 148 | * * 149 | ************************************************************/ 150 | static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr) 151 | { 152 | char *token; 153 | char *split; 154 | char *value, *n; 155 | Attribute_t *attr_split; 156 | GSList *the_list = attr_list; 157 | 158 | 159 | /* check for line continuation (+) */ 160 | if (strncmp(attr->value, "+", 1) == 0) attr->value++; 161 | /* check for end-of-line comments */ 162 | n = index(attr->value, '#'); 163 | /* if there is no comment check for trailing \n */ 164 | if(n == NULL) n = index(attr->value, '\n'); 165 | /* now copy the clean value into the attribute */ 166 | if(n == NULL) value = g_strdup(attr->value); 167 | else value = g_strndup(attr->value, (n - attr->value)); 168 | 169 | token=value; 170 | while((split=strsep(&token, ATTR_DELIMITERS))){ 171 | attr_split = attribute_new1(attr->type, split); 172 | if (attr_split) the_list = g_slist_append(the_list, attr_split); 173 | } 174 | free(value); 175 | attribute_free(attr, NULL); 176 | 177 | return(the_list); 178 | } 179 | 180 | /* XXX */ 181 | static void each_attribute_print(void *element_data, void *tr_ptr) 182 | { 183 | 184 | Attribute_t *attr = (Attribute_t *)element_data; 185 | 186 | fprintf(stderr, "[%d|%s]\n", attr->type, attr->value); 187 | 188 | } 189 | 190 | /* XXX */ 191 | static void print_object(Object_t *obj) 192 | { 193 | g_slist_foreach(obj->attributes, each_attribute_print, NULL); 194 | fprintf(stderr, ">>>>>\n%s\n", obj->object->str); 195 | } 196 | 197 | 198 | /****************************************************************** 199 | * GString *escape_apostrophes() * 200 | * Escapes apostrophes in the text so they do not confuse printf * 201 | * functions and don't corrupt SQL queries * 202 | * * 203 | * *****************************************************************/ 204 | static GString *escape_apostrophes(GString *text) { 205 | int i; 206 | for (i=0; i < text->len; i++) { 207 | if ((text->str[i] == '\'') || (text->str[i] == '\\')) { 208 | text = g_string_insert_c(text, i, '\\'); 209 | i++; 210 | } 211 | } 212 | return(text); 213 | } /* escape_apostrophes() */ 214 | 215 | 216 | /****************************************************************** 217 | * Line_Type_t line_type(e) * 218 | * Determines the line type analysing the first letters * 219 | * * 220 | * ****************************************************************/ 221 | static Line_Type_t line_type(const char *line, long *transaction_id) { 222 | 223 | if (strncmp(line, "# EOF", 4) == 0) return(LINE_EOF); 224 | if (strncmp(line, "#", 1) == 0) return(LINE_COMMENT); 225 | if (strcmp(line, "\n") == 0) return(LINE_EMPTY); 226 | 227 | if (strncmp(line, "ACK", 3) == 0) { 228 | *transaction_id = atol(line+3); 229 | return(LINE_ACK); 230 | } 231 | if (strncmp(line, "ADD_OVERRIDE", 12) == 0) { 232 | *transaction_id = atol(line+12); 233 | return(LINE_OVERRIDE_ADD); 234 | } 235 | if (strncmp(line, "UPD_OVERRIDE", 12) == 0) { 236 | *transaction_id = atol(line+12); 237 | return(LINE_OVERRIDE_UPD); 238 | } 239 | if (strncmp(line, "DEL_OVERRIDE", 12) == 0) { 240 | *transaction_id = atol(line+12); 241 | return(LINE_OVERRIDE_DEL); 242 | } 243 | 244 | if (strncmp(line, "ADD", 3) == 0) { 245 | *transaction_id = atol(line+3); 246 | return(LINE_ADD); 247 | } 248 | if (strncmp(line, "UPD", 3) == 0) { 249 | *transaction_id = atol(line+3); 250 | return(LINE_UPD); 251 | } 252 | if (strncmp(line, "DEL", 3) == 0) { 253 | *transaction_id = atol(line+3); 254 | return(LINE_DEL); 255 | } 256 | 257 | /* Otherwise this is an attribute */ 258 | return(LINE_ATTRIBUTE); 259 | 260 | } /* line_type() */ 261 | 262 | /****************************************************************** 263 | * Object_t *UD_parse_object() * 264 | * * 265 | * Parses the object accepting line by line * 266 | * Stores the object and list of attributes in Obj_parse_t struct * 267 | * * 268 | * ****************************************************************/ 269 | static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff) 270 | { 271 | GString *g_line_buff; 272 | Attribute_t *attr; 273 | 274 | /* in case of garbage */ 275 | if(parse->garbage) return(NULL); 276 | 277 | if(parse->obj==NULL) { 278 | /* which means this is the start - create a new object*/ 279 | parse->obj = object_new(line_buff); 280 | /* if this is not a class attribute - this is a garbage till the next blank line */ 281 | if(parse->obj==NULL) { 282 | parse->garbage = 1; 283 | return(NULL); 284 | } 285 | } 286 | 287 | /* allocate buffer or the string */ 288 | if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){ 289 | ER_perror(FAC_UD, UD_MEM, "cannot allocate gstring"); 290 | die; 291 | } 292 | /* copy the input */ 293 | g_string_sprintf(g_line_buff, "%s", line_buff); 294 | /* escape apostrophes in the input line */ 295 | g_line_buff=escape_apostrophes(g_line_buff); 296 | 297 | attr = attribute_new(g_line_buff->str); 298 | if (attr){ 299 | /* save this attribute for future line continuations */ 300 | /* skip attributes that are not stored in SQL database */ 301 | if(DF_get_update_query_type(attr->type)==UD_NULL_){ 302 | parse->current_attr = NULL; 303 | attribute_free(attr, NULL); 304 | } 305 | else { 306 | parse->current_attr = attr; 307 | if((attr->type == A_MB) || (attr->type == A_NH)) { 308 | /* insert it at the beginning */ 309 | /* mnt-by should go before member-of to allow correct membership autorization (still done in RIPupd) */ 310 | /* nic-hdl should go before any admin-c, tech-c to prevent errrors in self referencing role objects */ 311 | parse->obj->attributes = g_slist_insert(parse->obj->attributes, parse->current_attr, 1); 312 | } else { 313 | /* append it to the attribute list */ 314 | parse->obj->attributes = g_slist_append(parse->obj->attributes, parse->current_attr); 315 | } 316 | } 317 | } 318 | /* if this is not a valid attribute, then this may be unknown attr or line continuation */ 319 | else 320 | if(line_continuation(g_line_buff->str)) 321 | { 322 | /* if parse->attr == NULL, then this attr is not stored in SQL database - skip it */ 323 | if(parse->current_attr){ 324 | /* this is continuation of a previous attribute */ 325 | char *line_value, *new_value; 326 | char *n; 327 | /* normalize and append to current_attr->value */ 328 | /* check for end-of-line comments */ 329 | n = index(g_line_buff->str, '#'); 330 | /* if there is no comment check for trailing \n */ 331 | if(n == NULL) n = index(g_line_buff->str, '\n'); 332 | /* now copy the clean value into the attribute */ 333 | if(n == NULL) line_value = g_strdup(g_line_buff->str); 334 | else line_value = g_strndup(g_line_buff->str, (n - g_line_buff->str)); 335 | /* replace line continuation character with ' ' */ 336 | *line_value = ' '; 337 | new_value = g_strconcat(parse->current_attr->value, line_value, NULL); 338 | parse->current_attr = attribute_upd(parse->current_attr, parse->current_attr->type, new_value); 339 | UT_free(new_value); 340 | UT_free(line_value); 341 | } 342 | } 343 | /* otherwise, this is unknown attribute, so we need to break the continuation */ 344 | else parse->current_attr=NULL; 345 | /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */ 346 | g_string_sprintfa(parse->obj->object, "%s", g_line_buff->str); 347 | g_string_free(g_line_buff, TRUE); 348 | return(parse->obj); 349 | } 350 | 351 | /****************************************************************** 352 | * int each_attribute_2_pass() * 353 | * * 354 | * Assigns nic-handles in case of AUTO * 355 | * Splits attributes * 356 | * * * 357 | ******************************************************************/ 358 | void each_attribute_2_pass(void *element_data, void *ptr) 359 | { 360 | Attribute_t *attr = (Attribute_t *)element_data; 361 | Obj_parse_t *parse = (Obj_parse_t *)ptr; 362 | 363 | 364 | /* Strip the white space */ 365 | g_strstrip(attr->value); 366 | 367 | switch(attr->type){ 368 | case A_NH: /* nic-hdl */ 369 | /* Parse the string into nh structure */ 370 | /* In case of an AUTO NIC handle check the ID in the database */ 371 | /* Possible errors leave to core processing */ 372 | if(NH_parse(attr->value, &parse->nh_ptr) == 0) { 373 | /* this is an AUTO nic handle */ 374 | /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsing nic handle: [%s]", UD_TAG, attr->value);*/ 375 | /* Check if we can allocate it */ 376 | if(NH_check(parse->nh_ptr, parse->sql_connection)>0){ 377 | /* Convert nh to the database format */ 378 | parse->nic = NH_convert(parse->nh_ptr); 379 | /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsed and converted nic handle: [%s]", UD_TAG, nic); */ 380 | /* replace the auto nic with a real one */ 381 | if(ud_replace_substring(parse->obj->object, attr->value, parse->nic)==-1) { 382 | ER_perror(FAC_UD, UD_BUG, "cannot replace auto nic handle"); 383 | die; 384 | } 385 | /* Update the attribute */ 386 | attr = attribute_upd(attr, attr->type, parse->nic); 387 | 388 | } 389 | } 390 | parse->new_attr_list = g_slist_append(parse->new_attr_list, attr); 391 | break; 392 | case A_PN: /* person */ 393 | case A_RO: /* role */ 394 | case A_MR: /* mbrs-by-ref */ 395 | case A_MB: /* mnt-by */ 396 | case A_ML: /* mnt-lower */ 397 | case A_MO: /* member-of */ 398 | case A_SD: /* sub-dom */ 399 | case A_RZ: /* rev-srv */ 400 | case A_NS: /* nserver */ 401 | parse->new_attr_list = ud_split_attribute(parse->new_attr_list, attr); 402 | break; 403 | default: 404 | parse->new_attr_list = g_slist_append(parse->new_attr_list, attr); 405 | break; 406 | } 407 | 408 | } 409 | 410 | 411 | /****************************************************************** 412 | * int ud_normalize() * 413 | * * 414 | * function accepts Obj_parse_t structure with attribute list * 415 | * * 416 | * It makes another pass to normalize object * 417 | * Normalization includes: * 418 | * * 419 | * 1) reorder attributes * 420 | * . mnt-by should go before member-of to allow correct * 421 | * membership autorization (still done in RIPupd) * 422 | * . nic-hdl should go before any admin-c, tech-c to prevent * 423 | * errrors in self referencing role objects * 424 | * 2) in case of pn/ro check nic handle and replace it in * 425 | * the obj->object if needed (AUTO) * 426 | * 3) split some attributes that allow lists * 427 | ******************************************************************/ 428 | static int ud_normalize(Obj_parse_t *parse) 429 | { 430 | GSList *old_attr_list; 431 | GSList *first_element; 432 | Attribute_t *class_attr; 433 | 434 | old_attr_list = parse->obj->attributes; 435 | /* get class attribute - the first one */ 436 | first_element = g_slist_nth(old_attr_list, 0); 437 | class_attr = (Attribute_t *)first_element->data; 438 | /* save object name for reporting and checking ref integrity for names (legacy) */ 439 | parse->object_name = g_strdup(class_attr->value); 440 | /* reorder the attributes mnt-by and nic-hdl come first */ 441 | /*g_slist_foreach(old_attr_list, each_attribute_1_pass, parse); 442 | g_slist_free(old_attr_list); 443 | old_attr_list = parse->new_attr_list; */ 444 | parse->new_attr_list = NULL; 445 | /* assign auto nic-hdl and slit attributes */ 446 | g_slist_foreach(old_attr_list, each_attribute_2_pass, parse); 447 | g_slist_free(old_attr_list); 448 | parse->obj->attributes = parse->new_attr_list; 449 | 450 | return(1); 451 | } 452 | 453 | 454 | /****************************************************************** 455 | * report_transaction() * 456 | * * 457 | * Prints error report to the log * 458 | * * 459 | * reason - additional message that will be included * 460 | * * 461 | * *****************************************************************/ 462 | static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason) 463 | { 464 | int result=0; 465 | ut_timer_t fotime; 466 | float timediff; 467 | const char *class_name = DF_class_type2name(tr->class_type); 468 | char *primary_key = tr->K->str; 469 | 470 | 471 | /* calculate statistics */ 472 | UT_timeget(&fotime); 473 | timediff = UT_timediff(psotime, &fotime); 474 | 475 | if(tr->succeeded==0) { 476 | result=tr->error; 477 | log->num_failed++; 478 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] %.2fs FAILED [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason); 479 | /* ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] object: FAILED [%s][%s](%d/%d)", transaction_id, , reason, log->num_failed, (log->num_failed)+(log->num_ok)); */ 480 | if(result & ERROR_U_OBJ) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: referential integrity error", transaction_id); 481 | if(result & ERROR_U_AUT) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: authentication error", transaction_id); 482 | if(result & ERROR_U_BADOP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: unsupported operation", transaction_id); 483 | if(result & ERROR_U_COP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: conflicting operation", transaction_id); 484 | if(result & ERROR_U_NSUP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: this type is not supported", transaction_id); 485 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str); 486 | result=1; /* # of failures */ 487 | } 488 | else { 489 | result=0; 490 | log->num_ok++; 491 | /* ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: OK [%s](%d/%d)", transaction_id, obj_name, log->num_ok, (log->num_failed)+(log->num_ok)); */ 492 | ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] %.2fs OK [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason); 493 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str); 494 | } 495 | 496 | return(result); 497 | }/* report_transaction() */ 498 | 499 | 500 | 501 | /************************************************************ 502 | * process_nrtm() * 503 | * * 504 | * Process object in NRTM client mode * 505 | * * 506 | * nrtm - pointer to _nrtm structure * 507 | * log - pointer to Log_t structure * 508 | * object_name - name of the object * 509 | * operation - operation code (OP_ADD/OP_DEL) * 510 | * * 511 | * Returns: * 512 | * 1 - okay * 513 | * <0 - error * 514 | * * 515 | ************************************************************/ 516 | 517 | static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation) 518 | { 519 | int result=0; 520 | int dummy=0; 521 | struct _nrtm *nrtm = ud_stream->nrtm; 522 | long serial_id; 523 | Log_t *log_ptr= &(ud_stream->log); 524 | ut_timer_t sotime; 525 | int ta_upd_nhr; 526 | 527 | /* Start timer for statistics */ 528 | UT_timeget(&sotime); 529 | 530 | /* We allow NRTM updates for some inconsistent objects */ 531 | /* One of the examples is reference by name which looks like nic-handle */ 532 | /* For this purpose we allow dummy creation when updating an object */ 533 | /* We also check for dummy allowance when deleting an object */ 534 | /* this is done to allow deletion of person objects referenced by name */ 535 | 536 | tr->mode|=B_DUMMY; 537 | if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR; 538 | 539 | switch (operation) { 540 | 541 | case OP_ADD: 542 | if(nrtm->tr){ /* DEL ADD => saved*/ 543 | if(tr->object_id==0) { 544 | /* object does not exist in the DB */ 545 | /* delete the previous(saved) object*/ 546 | object_process(nrtm->tr); 547 | /* create DEL serial */ 548 | UD_lock_serial(nrtm->tr); 549 | serial_id = UD_create_serial(nrtm->tr); 550 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr); 551 | UD_commit_serial(nrtm->tr); 552 | UD_unlock_serial(nrtm->tr); 553 | /* Mark TR as clean */ 554 | TR_mark_clean(nrtm->tr); 555 | /* log the transaction */ 556 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL"); 557 | 558 | object_free(nrtm->tr->object); 559 | transaction_free(nrtm->tr); nrtm->tr=NULL; 560 | 561 | /* Create an object and update NHR */ 562 | tr->action=(TA_CREATE | ta_upd_nhr); 563 | /* restart the timer for statistics */ 564 | UT_timeget(&sotime); 565 | object_process(tr); /* create a new one*/ 566 | /* create ADD serial */ 567 | UD_lock_serial(tr); 568 | serial_id = UD_create_serial(tr); 569 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr); 570 | UD_commit_serial(tr); 571 | UD_unlock_serial(tr); 572 | /* Mark TR as clean */ 573 | TR_mark_clean(tr); 574 | /* log the transaction */ 575 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD"); 576 | } 577 | else { 578 | /* object already exists in the DB - update or dummy replacement*/ 579 | /*compare the two, may be we may collapse operations*/ 580 | if(tr->object_id==nrtm->tr->object_id) { 581 | /* DEL-ADD ->> UPDATE */ 582 | object_free(nrtm->tr->object); 583 | transaction_free(nrtm->tr); nrtm->tr=NULL; 584 | tr->action=TA_UPD_CLLPS; 585 | object_process(tr); 586 | /* create DEL+ADD serial records */ 587 | UD_lock_serial(tr); 588 | tr->action=TA_DELETE; serial_id = UD_create_serial(tr); 589 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL(UPD)"); 590 | 591 | /* restart the timer for statistics */ 592 | UT_timeget(&sotime); 593 | tr->sequence_id++; 594 | tr->action=TA_CREATE; serial_id = UD_create_serial(tr); 595 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr); 596 | UD_commit_serial(tr); 597 | UD_unlock_serial(tr); 598 | /* Mark TR as clean */ 599 | TR_mark_clean(tr); 600 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD(UPD)"); 601 | } 602 | else { /* this should be a dummy object in the database(that we are going to replace with the real one */ 603 | /* or an interleaved operation*/ 604 | object_process(nrtm->tr); /* delete the previous(saved) object*/ 605 | /* create a DEL serial record */ 606 | UD_lock_serial(nrtm->tr); 607 | serial_id = UD_create_serial(nrtm->tr); 608 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr); 609 | UD_commit_serial(nrtm->tr); 610 | UD_unlock_serial(nrtm->tr); 611 | /* Mark TR as clean */ 612 | TR_mark_clean(nrtm->tr); 613 | /* result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");*/ 614 | /* log the transaction */ 615 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL"); 616 | 617 | 618 | object_free(nrtm->tr->object); 619 | transaction_free(nrtm->tr); nrtm->tr=NULL; 620 | 621 | /* restart the timer for statistics */ 622 | UT_timeget(&sotime); 623 | 624 | tr->action=TA_UPDATE; 625 | /* check if we are replacing a dummy object */ 626 | dummy=isdummy(tr); 627 | /* If we are replacing dummy with a real object update NHR */ 628 | if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY); 629 | /* fprintf(stderr,"UPDATE next(dummy)\n"); */ 630 | object_process(tr); /* create a new one*/ 631 | /* result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new"); */ 632 | /* For serials this is CREATE operation. Increase sequence to have it correct in serals */ 633 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; } 634 | /* create ADD serial record */ 635 | UD_lock_serial(tr); 636 | serial_id = UD_create_serial(tr); 637 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr); 638 | UD_commit_serial(tr); 639 | UD_unlock_serial(tr); 640 | /* Mark TR as clean */ 641 | TR_mark_clean(tr); 642 | /* log the transaction */ 643 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD"); 644 | 645 | } 646 | } 647 | } 648 | else { /* ADD ADD =>brand new object*/ 649 | if(tr->object_id==0) { 650 | /* fprintf(stderr,"CREATE new\n");*/ 651 | /* Create an object and update NHR */ 652 | tr->action=(TA_CREATE | ta_upd_nhr); 653 | object_process(tr); 654 | /* create ADD serial */ 655 | UD_lock_serial(tr); 656 | serial_id = UD_create_serial(tr); 657 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr); 658 | UD_commit_serial(tr); 659 | UD_unlock_serial(tr); 660 | 661 | /* Mark TR as clean */ 662 | TR_mark_clean(tr); 663 | /* log the transaction */ 664 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD"); 665 | } 666 | else { /* object already exists in the database */ 667 | /* this may happen because of dummies*/ 668 | /* or with some implementations of mirroring protocol that have atomic update */ 669 | /* instead of add + del */ 670 | tr->action=TA_UPDATE; 671 | dummy=isdummy(tr); 672 | /* If we are replacing dummy with a real object update NHR */ 673 | if(dummy==1) tr->action = ( ta_upd_nhr| TA_UPD_DUMMY); 674 | object_process(tr); 675 | /* For serials this is CREATE operation. Increase sequence to have it correct in serals */ 676 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; } 677 | /* create ADD serial record */ 678 | UD_lock_serial(tr); 679 | serial_id = UD_create_serial(tr); 680 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr); 681 | UD_commit_serial(tr); 682 | UD_unlock_serial(tr); 683 | /* Mark TR as clean */ 684 | TR_mark_clean(tr); 685 | /* log the transaction */ 686 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD"); 687 | 688 | } 689 | } 690 | break; 691 | 692 | case OP_DEL: 693 | if(nrtm->tr){ /*DEL DEL =>saved */ 694 | object_process(nrtm->tr); /* delete the previous(saved) object*/ 695 | /* create DEL serial record */ 696 | UD_lock_serial(nrtm->tr); 697 | serial_id = UD_create_serial(nrtm->tr); 698 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr); 699 | UD_commit_serial(nrtm->tr); 700 | UD_unlock_serial(nrtm->tr); 701 | /* Mark TR as clean */ 702 | TR_mark_clean(nrtm->tr); 703 | /* log the transaction */ 704 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL"); 705 | 706 | object_free(nrtm->tr->object); 707 | transaction_free(nrtm->tr); nrtm->tr=NULL; 708 | } 709 | /* save the real object (not a dummy one ) */ 710 | if(tr->object_id>0 && !isdummy(tr)){ 711 | /* save the object*/ 712 | tr->action=(TA_DELETE | ta_upd_nhr); 713 | nrtm->tr=tr; 714 | return(0); 715 | } 716 | else { /* this is an error - Trying to DEL non-existing object*/ 717 | tr->succeeded=0; tr->error|=ERROR_U_COP; 718 | tr->action=(TA_DELETE | ta_upd_nhr); 719 | /* create and initialize TR record for crash recovery */ 720 | TR_create_record(tr); 721 | /* create DEL serial record anyway */ 722 | UD_lock_serial(tr); 723 | serial_id = UD_create_serial(tr); 724 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr); 725 | UD_commit_serial(tr); 726 | UD_unlock_serial(tr); 727 | /* Mark TR as clean */ 728 | TR_mark_clean(tr); 729 | /* log the transaction */ 730 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL: non-existing object"); 731 | 732 | } 733 | break; 734 | 735 | default: 736 | tr->succeeded=0; tr->error |=ERROR_U_BADOP; 737 | break; 738 | } 739 | 740 | /* Free resources */ 741 | object_free(tr->object); 742 | transaction_free(tr); 743 | 744 | return(result); 745 | } /* process_nrtm() */ 746 | 747 | 748 | 749 | /************************************************************ 750 | * process_updates() * 751 | * * 752 | * Process object in update mode * 753 | * * 754 | * ud_stream - pointer to UD_stream structure * 755 | * object_name - name of the object * 756 | * operation - operation code (OP_ADD/OP_DEL) * 757 | * * 758 | * Note: * 759 | * Frees tr and tr->obj on exit * 760 | * * 761 | * Returns: * 762 | * 0 - okay * 763 | * <0- number of failed objects * 764 | * * 765 | ************************************************************/ 766 | 767 | static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation) 768 | { 769 | int result=0; 770 | Log_t *log_ptr= &(ud_stream->log); 771 | int dummy=0; 772 | ut_timer_t sotime; 773 | int ta_upd_nhr; 774 | char *reason; 775 | 776 | /* Start timer for statistics */ 777 | UT_timeget(&sotime); 778 | if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR; 779 | 780 | switch(operation) { 781 | /* Compare operations and report an error if they do not match */ 782 | case OP_ADD: 783 | if(tr->object_id!=0) { /* trying to create, but object exists */ 784 | tr->succeeded=0; tr->error|=ERROR_U_COP; 785 | reason="U:ADD:object already exists"; 786 | UD_ack(tr); /* Send a NACK */ 787 | } else { 788 | /* Action: create the object and update NHR */ 789 | tr->action=(TA_CREATE | ta_upd_nhr); 790 | reason="U:ADD"; 791 | object_process(tr); 792 | } 793 | break; 794 | case OP_UPD: 795 | if(tr->object_id==0) { /* trying to update non-existing object*/ 796 | tr->succeeded=0; tr->error|=ERROR_U_COP; 797 | reason="U:UPD:non-existing object"; 798 | UD_ack(tr); /* Send a NACK */ 799 | } else { 800 | tr->action=TA_UPDATE; 801 | reason="U:UPD"; 802 | dummy=isdummy(tr); 803 | /* If we are replacing dummy with a real object update NHR */ 804 | if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY); 805 | object_process(tr); 806 | } 807 | break; 808 | 809 | case OP_DEL: 810 | if(tr->object_id==0) { /* trying t delete non-existing object*/ 811 | tr->succeeded=0; tr->error|=ERROR_U_COP; 812 | reason="U:DEL:non-existing object"; 813 | UD_ack(tr); 814 | } else { 815 | tr->action=(TA_DELETE | ta_upd_nhr); 816 | reason="U:DEL"; 817 | object_process(tr); 818 | } 819 | break; 820 | 821 | default: 822 | /* bad operation for this mode if not standalone */ 823 | if(IS_STANDALONE(tr->mode)) { 824 | if(tr->object_id==0){ 825 | tr->action=(TA_CREATE | ta_upd_nhr); 826 | reason="U:ADD"; 827 | } 828 | else { 829 | tr->action=TA_UPDATE; 830 | reason="U:UPD"; 831 | } 832 | object_process(tr); 833 | } 834 | else { 835 | tr->succeeded=0; 836 | tr->error|=ERROR_U_BADOP; 837 | reason="U:bad operation"; 838 | UD_ack(tr); /* Send a NACK */ 839 | } 840 | break; 841 | } 842 | /* If not in standalone mode create serial and copy error transcript */ 843 | if(!IS_STANDALONE(tr->mode)) { 844 | if(tr->succeeded){ 845 | /* we don't want to generate DEL serial for dummy replacement*/ 846 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; } 847 | UD_lock_serial(tr); 848 | UD_create_serial(tr); 849 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr); 850 | UD_commit_serial(tr); 851 | UD_unlock_serial(tr); 852 | /* Mark the TR as clean */ 853 | TR_mark_clean(tr); 854 | } 855 | } 856 | 857 | /* Make a report. U stands for update stream. No reason */ 858 | result=report_transaction(tr, tr->transaction_id, log_ptr, &sotime, reason); 859 | 860 | /* Free resources */ 861 | object_free(tr->object); 862 | transaction_free(tr); 863 | 864 | return(result); 865 | 866 | } /* process_updates() */ 867 | 868 | 869 | /************************************************************ 870 | * * 871 | * int process_transaction() * 872 | * * 873 | * Processes the transaction * 874 | * * 875 | * ud_stream - pointer to UD_stream_t structure * 876 | * * 877 | * Returns: * 878 | * 0 - no error * 879 | * <0- number of failed objects * 880 | * * 881 | ************************************************************/ 882 | 883 | /* It frees the obj */ 884 | 885 | static int process_transaction(UD_stream_t *ud_stream, 886 | Obj_parse_t *parse, 887 | int operation, 888 | long transaction_id) 889 | { 890 | Transaction_t *tr = NULL; 891 | Object_t *obj = parse->obj; 892 | int result; 893 | 894 | /* check if the requested transaction has already been processed */ 895 | /* this may happen in case of crash. If so, just send an ack and return */ 896 | if(TR_check(ud_stream->db_connection, transaction_id, (ud_stream->condat).sock))return(1); 897 | 898 | /* start new transaction now */ 899 | tr = transaction_new(ud_stream->db_connection, obj->type); 900 | 901 | /* Return with error if transaction cannot be created */ 902 | if (tr == NULL) die; 903 | 904 | tr->mode=ud_stream->ud_mode; 905 | tr->load_pass=ud_stream->load_pass; 906 | tr->object=obj; 907 | tr->nh=parse->nh_ptr; 908 | tr->source_hdl=ud_stream->source_hdl; 909 | tr->socket=(ud_stream->condat).sock; 910 | tr->transaction_id=transaction_id; 911 | 912 | /* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */ 913 | if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; } 914 | 915 | /* For the first load pass we only create objects */ 916 | if(ud_stream->load_pass==1) { 917 | Object_t *obj=tr->object; 918 | /* still we need to fill tr->K (last.pkey) field */ 919 | g_slist_foreach(obj->attributes, ud_each_primary_key_select, tr); 920 | tr->object_id=0; 921 | } 922 | else tr->object_id=get_object_id(tr); 923 | 924 | /* Object cannot be retrieved */ 925 | if(tr->object_id==-1) { /* DB error*/ 926 | tr->succeeded=0; 927 | tr->error |= ERROR_U_DBS; 928 | ER_perror(FAC_UD, UD_SQL, "%s: Object cannot be retrieved", parse->object_name); 929 | die; 930 | transaction_free(tr); 931 | object_free(obj); 932 | } 933 | /* save the name of person/role as we need it for referential */ 934 | /* integrity check when deleting the object against names. */ 935 | /* This is needed to support legacy references by name rather */ 936 | /* then by nic_hdl */ 937 | if((tr->class_type==C_PN) || (tr->class_type==C_RO)){ 938 | /* Save the value */ 939 | tr->save=g_strdup(parse->object_name); 940 | /* attribute_free(attr, NULL); */ 941 | } 942 | 943 | /* Process transaction. tr and obj are freed inside the process_* functions */ 944 | 945 | if(IS_UPDATE(ud_stream->ud_mode)) 946 | /* We are in update mode */ 947 | result=process_updates(ud_stream, tr, operation); 948 | else 949 | /* We are in NRTM mode */ 950 | result=process_nrtm(ud_stream, tr, operation); 951 | 952 | return(result); 953 | 954 | } 955 | 956 | 957 | /************************************************************ 958 | * * 959 | * int UD_process_stream(UD_stream_t *ud_stream) * 960 | * * 961 | * Processes the stream * 962 | * * 963 | * ud_stream - pointer to UD_stream_t structure * 964 | * * 965 | * Returns: * 966 | * in update mode (!standalone)(1 object processed): * 967 | * 1 - no error * 968 | * <0- errors * 969 | * * 970 | * in NRTM & standalone modes * 971 | * total number of object processed * 972 | * * 973 | ************************************************************/ 974 | 975 | int UD_process_stream(UD_stream_t *ud_stream) 976 | { 977 | char line_buff[STR_XXL]; 978 | Object_t *obj = NULL; 979 | SQ_connection_t *sql_connection; 980 | int start_object; 981 | int a_type; 982 | struct _nrtm *nrtm; 983 | Log_t *log_ptr= &(ud_stream->log); 984 | ut_timer_t stime, ftime, sotime; 985 | float obj_second1, obj_second10, timediff; 986 | int result; 987 | int operation=0; 988 | int interrupt=0; 989 | int do_update; 990 | int default_ud_mode = ud_stream->ud_mode; 991 | Line_Type_t linetype; 992 | Transaction_t *tr; 993 | long transaction_id=0; /* transaction_id (to be supplied by DBupdate and stored in Database) */ 994 | long serial_id; 995 | Obj_parse_t obj_parse; /* the structure used to parse a text object */ 996 | 997 | 998 | 999 | 1000 | nrtm=ud_stream->nrtm; 1001 | start_object = 1; 1002 | a_type=-1; 1003 | 1004 | 1005 | /* Check connection to the database */ 1006 | if(SQ_ping(ud_stream->db_connection)) { 1007 | ER_perror(FAC_UD, UD_SQL, "%s", SQ_error(ud_stream->db_connection)); 1008 | die; 1009 | } 1010 | 1011 | sql_connection=ud_stream->db_connection; 1012 | 1013 | ud_parse_init(sql_connection, &obj_parse); 1014 | /* Start timer for statistics */ 1015 | UT_timeget(&stime); 1016 | 1017 | /* Main loop. Reading input stream line by line */ 1018 | /* Empty line signals to start processing an object, if we have it */ 1019 | while (SK_cd_gets(&ud_stream->condat, line_buff, sizeof(line_buff))>0) { 1020 | 1021 | 1022 | switch (linetype=line_type(line_buff, &transaction_id)) { 1023 | case LINE_ATTRIBUTE: 1024 | /* parse the object line by line */ 1025 | obj = UD_parse_object(&obj_parse, line_buff); 1026 | 1027 | break; 1028 | 1029 | case LINE_COMMENT: 1030 | break; 1031 | 1032 | case LINE_EOF: 1033 | break; 1034 | 1035 | case LINE_ACK: 1036 | tr = transaction_new(ud_stream->db_connection, 0); 1037 | tr->transaction_id=transaction_id; 1038 | TR_delete_record(tr); 1039 | transaction_free(tr); 1040 | break; 1041 | 1042 | 1043 | case LINE_ADD: 1044 | /* restore the default operation mode */ 1045 | operation=OP_ADD; 1046 | ud_stream->ud_mode=default_ud_mode; 1047 | break; 1048 | 1049 | case LINE_OVERRIDE_ADD: 1050 | /* for override - switch the dummy bit on */ 1051 | operation=OP_ADD; 1052 | ud_stream->ud_mode=default_ud_mode|B_DUMMY; 1053 | break; 1054 | 1055 | case LINE_UPD: 1056 | /* restore the default operation mode */ 1057 | operation=OP_UPD; 1058 | ud_stream->ud_mode=default_ud_mode; 1059 | break; 1060 | 1061 | case LINE_OVERRIDE_UPD: 1062 | /* for override - switch the dummy bit on */ 1063 | operation=OP_UPD; 1064 | ud_stream->ud_mode=default_ud_mode|B_DUMMY; 1065 | break; 1066 | 1067 | case LINE_DEL: 1068 | /* restore the default operation mode */ 1069 | operation=OP_DEL; 1070 | ud_stream->ud_mode=default_ud_mode; 1071 | break; 1072 | 1073 | case LINE_OVERRIDE_DEL: 1074 | /* for override - switch the dummy bit on */ 1075 | operation=OP_DEL; 1076 | ud_stream->ud_mode=default_ud_mode|B_DUMMY; 1077 | break; 1078 | 1079 | case LINE_EMPTY: 1080 | /* start processing the object */ 1081 | if ((obj=obj_parse.obj)) { /* if not just garbage*/ 1082 | /* normalize the object (reorder and split attributes */ 1083 | ud_normalize(&obj_parse); 1084 | /* XXX */ 1085 | /* print_object(obj); */ 1086 | 1087 | /* start new transaction now */ 1088 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: [%s] ", transaction_id, obj_parse.object_name); 1089 | 1090 | /* fprintf(stderr, "transction # %ld\n", transaction_id); */ 1091 | result=process_transaction(ud_stream, &obj_parse, operation, transaction_id); 1092 | /* process_transaction() frees tr and obj structures, */ 1093 | /* so make sure we'll not reference these objects in the future */ 1094 | operation=OP_NOOP; 1095 | transaction_id=0; 1096 | ud_stream->ud_mode=default_ud_mode; 1097 | ud_parse_free(&obj_parse); 1098 | 1099 | /* this is a good place for quick interrupt */ 1100 | do_update=CO_get_do_update(); 1101 | if (do_update) interrupt=0; else interrupt=1; 1102 | } /* if this is a real object */ 1103 | /* initialize the parsing structure */ 1104 | ud_parse_init(sql_connection, &obj_parse); 1105 | 1106 | break; 1107 | 1108 | default: 1109 | die; 1110 | } /* switch */ 1111 | 1112 | /* Finish processing if interrupt has been set */ 1113 | if (interrupt) break; 1114 | } /* Main loop of data stream processing : while */ 1115 | 1116 | /* Some postprocessing */ 1117 | if(IS_NRTM_CLNT(ud_stream->ud_mode)){ 1118 | /* We are in NRTM mode */ 1119 | /* Clean up */ 1120 | /* fclose(ud_stream->stream); */ 1121 | /* In NRTM mode there may be a saved object that is unprocessed */ 1122 | if(nrtm->tr){ /*saved backlog?*/ 1123 | /* restart the timer for statistics */ 1124 | UT_timeget(&sotime); 1125 | object_process(nrtm->tr); /* delete the previous(saved) object*/ 1126 | /* result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name, 1127 | "NRTM:DEL:While deleting previous(saved) object"); */ 1128 | /* create DEL serial record no matter what the result is */ 1129 | UD_lock_serial(nrtm->tr); 1130 | serial_id = UD_create_serial(nrtm->tr); 1131 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr); 1132 | UD_commit_serial(nrtm->tr); 1133 | UD_unlock_serial(nrtm->tr); 1134 | /* Mark TR as clean */ 1135 | TR_mark_clean(nrtm->tr); 1136 | /* log the transaction */ 1137 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL"); 1138 | 1139 | object_free(nrtm->tr->object); 1140 | transaction_free(nrtm->tr); nrtm->tr=NULL; 1141 | } 1142 | } 1143 | 1144 | /* That's all. Free GString */ 1145 | /* g_string_free(g_line_buff, TRUE);*/ 1146 | 1147 | 1148 | /* Calculate some statistics */ 1149 | /* ftime=time(NULL); */ 1150 | UT_timeget(&ftime); 1151 | /* obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime); 1152 | obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime); */ 1153 | timediff = UT_timediff(&stime, &ftime); 1154 | obj_second1 = (float)(log_ptr->num_ok)/timediff; 1155 | obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/timediff; 1156 | 1157 | /* Print the report */ 1158 | if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) { 1159 | 1160 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s ******** report **********", UD_TAG); 1161 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects OK (%7.4f obj/s)", UD_TAG, log_ptr->num_ok, obj_second1); 1162 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects failed", UD_TAG, log_ptr->num_failed); 1163 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s average processing time %7.4f obj/s (%6.2f obj/min)", UD_TAG, 1164 | obj_second10, obj_second10*60); 1165 | result=log_ptr->num_ok+log_ptr->num_failed; 1166 | } 1167 | return(result); 1168 | 1169 | } /* UD_process_stream */ 1170 |