corosync  3.1.5
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2006 MontaVista Software, Inc.
3  * Copyright (c) 2006-2021 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
72 #include <config.h>
73 
74 #include <pthread.h>
75 #include <assert.h>
76 #include <sys/types.h>
77 #include <sys/file.h>
78 #include <sys/poll.h>
79 #include <sys/uio.h>
80 #include <sys/mman.h>
81 #include <sys/socket.h>
82 #include <sys/un.h>
83 #include <sys/time.h>
84 #include <sys/resource.h>
85 #include <sys/stat.h>
86 #include <netinet/in.h>
87 #include <arpa/inet.h>
88 #include <unistd.h>
89 #include <fcntl.h>
90 #include <stdlib.h>
91 #include <stdio.h>
92 #include <errno.h>
93 #include <signal.h>
94 #include <sched.h>
95 #include <time.h>
96 #include <semaphore.h>
97 #include <string.h>
98 
99 #ifdef HAVE_LIBSYSTEMD
100 #include <systemd/sd-daemon.h>
101 #endif
102 
103 #include <qb/qbdefs.h>
104 #include <qb/qblog.h>
105 #include <qb/qbloop.h>
106 #include <qb/qbutil.h>
107 #include <qb/qbipcs.h>
108 
109 #include <corosync/swab.h>
110 #include <corosync/corotypes.h>
111 #include <corosync/corodefs.h>
112 #include <corosync/totem/totempg.h>
113 #include <corosync/logsys.h>
114 #include <corosync/icmap.h>
115 
116 #include "quorum.h"
117 #include "totemsrp.h"
118 #include "logconfig.h"
119 #include "totemconfig.h"
120 #include "main.h"
121 #include "sync.h"
122 #include "timer.h"
123 #include "util.h"
124 #include "apidef.h"
125 #include "service.h"
126 #include "schedwrk.h"
127 #include "ipcs_stats.h"
128 #include "stats.h"
129 
130 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
131 #define IPC_LOGSYS_SIZE 1024*64
132 #else
133 #define IPC_LOGSYS_SIZE 8192*128
134 #endif
135 
136 /*
137  * LibQB adds default "*" syslog filter so we have to set syslog_priority as low
138  * as possible so filters applied later in _logsys_config_apply_per_file takes
139  * effect.
140  */
143  LOG_DAEMON,
144  LOG_EMERG);
145 
147 
148 #define SERVER_BACKLOG 5
149 
150 static int sched_priority = 0;
151 
152 static unsigned int service_count = 32;
153 
155 
156 static struct corosync_api_v1 *api = NULL;
157 
158 static int sync_in_process = 1;
159 
160 static qb_loop_t *corosync_poll_handle;
161 
162 struct sched_param global_sched_param;
163 
164 static corosync_timer_handle_t corosync_stats_timer_handle;
165 
166 static const char *corosync_lock_file = LOCALSTATEDIR"/run/corosync.pid";
167 
168 static char corosync_config_file[PATH_MAX + 1] = COROSYSCONFDIR "/corosync.conf";
169 
170 static int lockfile_fd = -1;
171 
176 };
177 
178 qb_loop_t *cs_poll_handle_get (void)
179 {
180  return (corosync_poll_handle);
181 }
182 
183 int cs_poll_dispatch_add (qb_loop_t * handle,
184  int fd,
185  int events,
186  void *data,
187 
188  int (*dispatch_fn) (int fd,
189  int revents,
190  void *data))
191 {
192  return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
193  dispatch_fn);
194 }
195 
196 int cs_poll_dispatch_delete(qb_loop_t * handle, int fd)
197 {
198  return qb_loop_poll_del(handle, fd);
199 }
200 
202 {
203  int i;
204 
205  for (i = 0; i < SERVICES_COUNT_MAX; i++) {
206  if (corosync_service[i] && corosync_service[i]->exec_dump_fn) {
208  }
209  }
210 }
211 
212 const char *corosync_get_config_file(void)
213 {
214 
215  return (corosync_config_file);
216 }
217 
218 static void corosync_blackbox_write_to_file (void)
219 {
220  char fname[PATH_MAX];
221  char fdata_fname[PATH_MAX];
222  char time_str[PATH_MAX];
223  struct tm cur_time_tm;
224  time_t cur_time_t;
225  ssize_t res;
226 
227  cur_time_t = time(NULL);
228  localtime_r(&cur_time_t, &cur_time_tm);
229 
230  strftime(time_str, PATH_MAX, "%Y-%m-%dT%H:%M:%S", &cur_time_tm);
231  if (snprintf(fname, PATH_MAX, "%s/fdata-%s-%lld",
232  get_state_dir(),
233  time_str,
234  (long long int)getpid()) >= PATH_MAX) {
235  log_printf(LOGSYS_LEVEL_ERROR, "Can't snprintf blackbox file name");
236  return ;
237  }
238 
239  if ((res = qb_log_blackbox_write_to_file(fname)) < 0) {
240  LOGSYS_PERROR(-res, LOGSYS_LEVEL_ERROR, "Can't store blackbox file");
241  return ;
242  }
243  snprintf(fdata_fname, sizeof(fdata_fname), "%s/fdata", get_state_dir());
244  unlink(fdata_fname);
245  if (symlink(fname, fdata_fname) == -1) {
246  log_printf(LOGSYS_LEVEL_ERROR, "Can't create symlink to '%s' for corosync blackbox file '%s'",
247  fname, fdata_fname);
248  }
249 }
250 
251 static void unlink_all_completed (void)
252 {
253  api->timer_delete (corosync_stats_timer_handle);
254  qb_loop_stop (corosync_poll_handle);
255  icmap_fini();
256 }
257 
259 {
260  corosync_service_unlink_all (api, unlink_all_completed);
261 }
262 
263 static int32_t sig_diag_handler (int num, void *data)
264 {
266  return 0;
267 }
268 
269 static int32_t sig_exit_handler (int num, void *data)
270 {
271  log_printf(LOGSYS_LEVEL_NOTICE, "Node was shut down by a signal");
272  corosync_service_unlink_all (api, unlink_all_completed);
273  return 0;
274 }
275 
276 static void sigsegv_handler (int num)
277 {
278  (void)signal (num, SIG_DFL);
279  corosync_blackbox_write_to_file ();
280  qb_log_fini();
281  raise (num);
282 }
283 
284 #define LOCALHOST_IP inet_addr("127.0.0.1")
285 
286 static void *corosync_group_handle;
287 
288 static struct totempg_group corosync_group = {
289  .group = "a",
290  .group_len = 1
291 };
292 
293 static void serialize_lock (void)
294 {
295 }
296 
297 static void serialize_unlock (void)
298 {
299 }
300 
301 static void corosync_sync_completed (void)
302 {
304  "Completed service synchronization, ready to provide service.");
305  sync_in_process = 0;
306 
307  cs_ipcs_sync_state_changed(sync_in_process);
309  /*
310  * Inform totem to start using new message queue again
311  */
313 
314 #ifdef HAVE_LIBSYSTEMD
315  sd_notify (0, "READY=1");
316 #endif
317 }
318 
319 static int corosync_sync_callbacks_retrieve (
320  int service_id,
321  struct sync_callbacks *callbacks)
322 {
323  if (corosync_service[service_id] == NULL) {
324  return (-1);
325  }
326 
327  if (callbacks == NULL) {
328  return (0);
329  }
330 
331  callbacks->name = corosync_service[service_id]->name;
332 
333  callbacks->sync_init = corosync_service[service_id]->sync_init;
334  callbacks->sync_process = corosync_service[service_id]->sync_process;
335  callbacks->sync_activate = corosync_service[service_id]->sync_activate;
336  callbacks->sync_abort = corosync_service[service_id]->sync_abort;
337  return (0);
338 }
339 
340 static struct memb_ring_id corosync_ring_id;
341 
342 static void member_object_joined (unsigned int nodeid)
343 {
344  char member_ip[ICMAP_KEYNAME_MAXLEN];
345  char member_join_count[ICMAP_KEYNAME_MAXLEN];
346  char member_status[ICMAP_KEYNAME_MAXLEN];
347 
348  snprintf(member_ip, ICMAP_KEYNAME_MAXLEN,
349  "runtime.members.%u.ip", nodeid);
350  snprintf(member_join_count, ICMAP_KEYNAME_MAXLEN,
351  "runtime.members.%u.join_count", nodeid);
352  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
353  "runtime.members.%u.status", nodeid);
354 
355  if (icmap_get(member_ip, NULL, NULL, NULL) == CS_OK) {
356  icmap_inc(member_join_count);
357  icmap_set_string(member_status, "joined");
358  } else {
359  icmap_set_string(member_ip, (char*)api->totem_ifaces_print (nodeid));
360  icmap_set_uint32(member_join_count, 1);
361  icmap_set_string(member_status, "joined");
362  }
363 
365  "Member joined: %s", api->totem_ifaces_print (nodeid));
366 }
367 
368 static void member_object_left (unsigned int nodeid)
369 {
370  char member_status[ICMAP_KEYNAME_MAXLEN];
371 
372  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
373  "runtime.members.%u.status", nodeid);
374  icmap_set_string(member_status, "left");
375 
377  "Member left: %s", api->totem_ifaces_print (nodeid));
378 }
379 
380 static void confchg_fn (
381  enum totem_configuration_type configuration_type,
382  const unsigned int *member_list, size_t member_list_entries,
383  const unsigned int *left_list, size_t left_list_entries,
384  const unsigned int *joined_list, size_t joined_list_entries,
385  const struct memb_ring_id *ring_id)
386 {
387  int i;
388  int abort_activate = 0;
389 
390  if (sync_in_process == 1) {
391  abort_activate = 1;
392  }
393  sync_in_process = 1;
394  cs_ipcs_sync_state_changed(sync_in_process);
395  memcpy (&corosync_ring_id, ring_id, sizeof (struct memb_ring_id));
396 
397  for (i = 0; i < left_list_entries; i++) {
398  member_object_left (left_list[i]);
399  }
400  for (i = 0; i < joined_list_entries; i++) {
401  member_object_joined (joined_list[i]);
402  }
403  /*
404  * Call configuration change for all services
405  */
406  for (i = 0; i < service_count; i++) {
407  if (corosync_service[i] && corosync_service[i]->confchg_fn) {
408  corosync_service[i]->confchg_fn (configuration_type,
409  member_list, member_list_entries,
410  left_list, left_list_entries,
411  joined_list, joined_list_entries, ring_id);
412  }
413  }
414 
415  if (abort_activate) {
416  sync_abort ();
417  }
418  if (configuration_type == TOTEM_CONFIGURATION_TRANSITIONAL) {
419  sync_save_transitional (member_list, member_list_entries, ring_id);
420  }
421  if (configuration_type == TOTEM_CONFIGURATION_REGULAR) {
422  sync_start (member_list, member_list_entries, ring_id);
423  }
424 }
425 
426 static void priv_drop (void)
427 {
428  return; /* TODO: we are still not dropping privs */
429 }
430 
431 static void corosync_tty_detach (void)
432 {
433  int devnull;
434 
435  /*
436  * Disconnect from TTY if this is not a debug run
437  */
438 
439  switch (fork ()) {
440  case -1:
442  break;
443  case 0:
444  /*
445  * child which is disconnected, run this process
446  */
447  break;
448  default:
449  exit (0);
450  break;
451  }
452 
453  /* Create new session */
454  (void)setsid();
455 
456  /*
457  * Map stdin/out/err to /dev/null.
458  */
459  devnull = open("/dev/null", O_RDWR);
460  if (devnull == -1) {
462  }
463 
464  if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
465  || dup2(devnull, 2) < 0) {
466  close(devnull);
468  }
469  close(devnull);
470 }
471 
472 static void corosync_mlockall (void)
473 {
474  int res;
475  struct rlimit rlimit;
476 
477  rlimit.rlim_cur = RLIM_INFINITY;
478  rlimit.rlim_max = RLIM_INFINITY;
479 
480 #ifndef RLIMIT_MEMLOCK
481 #define RLIMIT_MEMLOCK RLIMIT_VMEM
482 #endif
483 
484  res = setrlimit (RLIMIT_MEMLOCK, &rlimit);
485  if (res == -1) {
487  "Could not increase RLIMIT_MEMLOCK, not locking memory");
488  return;
489  }
490 
491  res = mlockall (MCL_CURRENT | MCL_FUTURE);
492  if (res == -1) {
494  "Could not lock memory of service to avoid page faults");
495  };
496 }
497 
498 
499 static void corosync_totem_stats_updater (void *data)
500 {
501  totempg_stats_t * stats;
502  uint32_t total_mtt_rx_token;
503  uint32_t total_backlog_calc;
504  uint32_t total_token_holdtime;
505  int t, prev;
506  int32_t token_count;
507  const char *cstr;
508 
509  stats = api->totem_get_stats();
510 
511 
513 
514  if (stats->srp->continuous_gather > MAX_NO_CONT_GATHER ||
516  cstr = "";
517 
519  cstr = "number of multicast sendmsg failures is above threshold";
520  }
521 
522  if (stats->srp->continuous_gather > MAX_NO_CONT_GATHER) {
523  cstr = "totem is continuously in gather state";
524  }
525 
527  "Totem is unable to form a cluster because of an "
528  "operating system or network fault (reason: %s). The most common "
529  "cause of this message is that the local firewall is "
530  "configured improperly.", cstr);
532  } else {
534  }
535 
536  total_mtt_rx_token = 0;
537  total_token_holdtime = 0;
538  total_backlog_calc = 0;
539  token_count = 0;
540  t = stats->srp->latest_token;
541  while (1) {
542  if (t == 0)
543  prev = TOTEM_TOKEN_STATS_MAX - 1;
544  else
545  prev = t - 1;
546  if (prev == stats->srp->earliest_token)
547  break;
548  /* if tx == 0, then dropped token (not ours) */
549  if (stats->srp->token[t].tx != 0 ||
550  (stats->srp->token[t].rx - stats->srp->token[prev].rx) > 0 ) {
551  total_mtt_rx_token += (stats->srp->token[t].rx - stats->srp->token[prev].rx);
552  total_token_holdtime += (stats->srp->token[t].tx - stats->srp->token[t].rx);
553  total_backlog_calc += stats->srp->token[t].backlog_calc;
554  token_count++;
555  }
556  t = prev;
557  }
558  if (token_count) {
559  stats->srp->mtt_rx_token = (total_mtt_rx_token / token_count);
560  stats->srp->avg_token_workload = (total_token_holdtime / token_count);
561  stats->srp->avg_backlog_calc = (total_backlog_calc / token_count);
562  }
563 
564  stats->srp->time_since_token_last_received = qb_util_nano_current_get () / QB_TIME_NS_IN_MSEC -
565  stats->srp->token[stats->srp->latest_token].rx;
566 
568 
569  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
570  corosync_totem_stats_updater,
571  &corosync_stats_timer_handle);
572 }
573 
574 static void corosync_totem_stats_init (void)
575 {
576  /* start stats timer */
577  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
578  corosync_totem_stats_updater,
579  &corosync_stats_timer_handle);
580 }
581 
582 static void deliver_fn (
583  unsigned int nodeid,
584  const void *msg,
585  unsigned int msg_len,
586  int endian_conversion_required)
587 {
588  const struct qb_ipc_request_header *header;
589  int32_t service;
590  int32_t fn_id;
591  uint32_t id;
592 
593  header = msg;
594  if (endian_conversion_required) {
595  id = swab32 (header->id);
596  } else {
597  id = header->id;
598  }
599 
600  /*
601  * Call the proper executive handler
602  */
603  service = id >> 16;
604  fn_id = id & 0xffff;
605 
606  if (!corosync_service[service]) {
607  return;
608  }
609  if (fn_id >= corosync_service[service]->exec_engine_count) {
610  log_printf(LOGSYS_LEVEL_WARNING, "discarded unknown message %d for service %d (max id %d)",
611  fn_id, service, corosync_service[service]->exec_engine_count);
612  return;
613  }
614 
615  icmap_fast_inc(service_stats_rx[service][fn_id]);
616 
617  if (endian_conversion_required) {
618  assert(corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
620  ((void *)msg);
621  }
622 
624  (msg, nodeid);
625 }
626 
628  const struct iovec *iovec,
629  unsigned int iov_len,
630  unsigned int guarantee)
631 {
632  const struct qb_ipc_request_header *req = iovec->iov_base;
633  int32_t service;
634  int32_t fn_id;
635 
636  service = req->id >> 16;
637  fn_id = req->id & 0xffff;
638 
639  if (corosync_service[service]) {
640  icmap_fast_inc(service_stats_tx[service][fn_id]);
641  }
642 
643  return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee));
644 }
645 
646 static void corosync_ring_id_create_or_load (
647  struct memb_ring_id *memb_ring_id,
648  unsigned int nodeid)
649 {
650  int fd;
651  int res = 0;
652  char filename[PATH_MAX];
653 
654  snprintf (filename, sizeof(filename), "%s/ringid_%u",
655  get_state_dir(), nodeid);
656  fd = open (filename, O_RDONLY);
657  /*
658  * If file can be opened and read, read the ring id
659  */
660  if (fd != -1) {
661  res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
662  close (fd);
663  }
664  /*
665  * If file could not be opened or read, create a new ring id
666  */
667  if ((fd == -1) || (res != sizeof (uint64_t))) {
668  memb_ring_id->seq = 0;
669  fd = creat (filename, 0600);
670  if (fd != -1) {
671  res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
672  close (fd);
673  if (res == -1) {
675  "Couldn't write ringid file '%s'", filename);
676 
678  }
679  } else {
681  "Couldn't create ringid file '%s'", filename);
682 
684  }
685  }
686 
688 }
689 
690 static void corosync_ring_id_store (
691  const struct memb_ring_id *memb_ring_id,
692  unsigned int nodeid)
693 {
694  char filename[PATH_MAX];
695  int fd;
696  int res;
697 
698  snprintf (filename, sizeof(filename), "%s/ringid_%u",
699  get_state_dir(), nodeid);
700 
701  fd = creat (filename, 0600);
702  if (fd == -1) {
704  "Couldn't store new ring id " CS_PRI_RING_ID_SEQ " to stable storage",
705  memb_ring_id->seq);
706 
708  }
710  "Storing new sequence id for ring " CS_PRI_RING_ID_SEQ, memb_ring_id->seq);
711  res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq));
712  close (fd);
713  if (res != sizeof(memb_ring_id->seq)) {
715  "Couldn't store new ring id " CS_PRI_RING_ID_SEQ " to stable storage",
716  memb_ring_id->seq);
717 
719  }
720 }
721 
722 static qb_loop_timer_handle recheck_the_q_level_timer;
724 {
725  totempg_check_q_level(corosync_group_handle);
727  qb_loop_timer_add(cs_poll_handle_get(), QB_LOOP_MED, 1*QB_TIME_NS_IN_MSEC,
728  NULL, corosync_recheck_the_q_level, &recheck_the_q_level_timer);
729  }
730 }
731 
734 };
735 
736 
738  unsigned int service,
739  unsigned int id,
740  const void *msg,
741  void *sending_allowed_private_data)
742 {
744  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
745  struct iovec reserve_iovec;
746  struct qb_ipc_request_header *header = (struct qb_ipc_request_header *)msg;
747  int sending_allowed;
748 
749  reserve_iovec.iov_base = (char *)header;
750  reserve_iovec.iov_len = header->size;
751 
753  corosync_group_handle,
754  &reserve_iovec, 1);
755  if (pd->reserved_msgs == -1) {
756  return -EINVAL;
757  }
758 
759  /* Message ID out of range */
760  if (id >= corosync_service[service]->lib_engine_count) {
761  return -EINVAL;
762  }
763 
764  sending_allowed = QB_FALSE;
765  if (corosync_quorum_is_quorate() == 1 ||
766  corosync_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) {
767  // we are quorate
768  // now check flow control
769  if (corosync_service[service]->lib_engine[id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) {
770  sending_allowed = QB_TRUE;
771  } else if (pd->reserved_msgs && sync_in_process == 0) {
772  sending_allowed = QB_TRUE;
773  } else if (pd->reserved_msgs == 0) {
774  return -ENOBUFS;
775  } else /* (sync_in_process) */ {
776  return -EINPROGRESS;
777  }
778  } else {
779  return -EHOSTUNREACH;
780  }
781 
782  return (sending_allowed);
783 }
784 
785 void corosync_sending_allowed_release (void *sending_allowed_private_data)
786 {
788  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
789 
790  if (pd->reserved_msgs == -1) {
791  return;
792  }
794 }
795 
797 {
798  int ret = 0;
799 
800  assert (source != NULL);
801  if (source->nodeid == totempg_my_nodeid_get ()) {
802  ret = 1;
803  }
804  return ret;
805 }
806 
808  mar_message_source_t *source,
809  void *conn)
810 {
811  assert ((source != NULL) && (conn != NULL));
812  memset (source, 0, sizeof (mar_message_source_t));
813  source->nodeid = totempg_my_nodeid_get ();
814  source->conn = conn;
815 }
816 
819  qb_loop_timer_handle handle;
820  unsigned long long tv_prev;
821  unsigned long long max_tv_diff;
822 };
823 
824 static void timer_function_scheduler_timeout (void *data)
825 {
826  struct scheduler_pause_timeout_data *timeout_data = (struct scheduler_pause_timeout_data *)data;
827  unsigned long long tv_current;
828  unsigned long long tv_diff;
829  uint64_t schedmiss_event_tstamp;
830 
831  tv_current = qb_util_nano_current_get ();
832 
833  if (timeout_data->tv_prev == 0) {
834  /*
835  * Initial call -> just pretent everything is ok
836  */
837  timeout_data->tv_prev = tv_current;
838  timeout_data->max_tv_diff = 0;
839  }
840 
841  tv_diff = tv_current - timeout_data->tv_prev;
842  timeout_data->tv_prev = tv_current;
843 
844  if (tv_diff > timeout_data->max_tv_diff) {
845  schedmiss_event_tstamp = qb_util_nano_from_epoch_get() / QB_TIME_NS_IN_MSEC;
846 
847  log_printf (LOGSYS_LEVEL_WARNING, "Corosync main process was not scheduled (@%" PRIu64 ") for %0.4f ms "
848  "(threshold is %0.4f ms). Consider token timeout increase.",
849  schedmiss_event_tstamp,
850  (float)tv_diff / QB_TIME_NS_IN_MSEC, (float)timeout_data->max_tv_diff / QB_TIME_NS_IN_MSEC);
851 
852  stats_add_schedmiss_event(schedmiss_event_tstamp, (float)tv_diff / QB_TIME_NS_IN_MSEC);
853  }
854 
855  /*
856  * Set next threshold, because token_timeout can change
857  */
858  timeout_data->max_tv_diff = timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC * 0.8;
859  qb_loop_timer_add (corosync_poll_handle,
860  QB_LOOP_MED,
861  timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC / 3,
862  timeout_data,
863  timer_function_scheduler_timeout,
864  &timeout_data->handle);
865 }
866 
867 
868 /*
869  * Set main pid RR scheduler.
870  * silent: don't log sched_get_priority_max and sched_setscheduler errors
871  * Returns: 0 - success, -1 failure, -2 platform doesn't support SCHED_RR
872  */
873 static int corosync_set_rr_scheduler (int silent)
874 {
875  int ret_val = 0;
876 
877 #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
878  int res;
879 
880  sched_priority = sched_get_priority_max (SCHED_RR);
881  if (sched_priority != -1) {
882  global_sched_param.sched_priority = sched_priority;
883  res = sched_setscheduler (0, SCHED_RR, &global_sched_param);
884  if (res == -1) {
885  if (!silent) {
887  "Could not set SCHED_RR at priority %d",
888  global_sched_param.sched_priority);
889  }
890 
891  global_sched_param.sched_priority = 0;
892 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
893  qb_log_thread_priority_set (SCHED_OTHER, 0);
894 #endif
895  ret_val = -1;
896  } else {
897 
898  /*
899  * Turn on SCHED_RR in logsys system
900  */
901 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
902  res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
903 #else
904  res = -1;
905 #endif
906  if (res == -1) {
908  "Could not set logsys thread priority."
909  " Can't continue because of priority inversions.");
911  }
912  }
913  } else {
914  if (!silent) {
916  "Could not get maximum scheduler priority");
917  }
918  sched_priority = 0;
919  ret_val = -1;
920  }
921 #else
923  "The Platform is missing process priority setting features. Leaving at default.");
924  ret_val = -2;
925 #endif
926 
927  return (ret_val);
928 }
929 
930 
931 /* The basename man page contains scary warnings about
932  thread-safety and portability, hence this */
933 static const char *corosync_basename(const char *file_name)
934 {
935  char *base;
936  base = strrchr (file_name, '/');
937  if (base) {
938  return base + 1;
939  }
940 
941  return file_name;
942 }
943 
944 static void
945 _logsys_log_printf(int level, int subsys,
946  const char *function_name,
947  const char *file_name,
948  int file_line,
949  const char *format,
950  ...) __attribute__((format(printf, 6, 7)));
951 
952 static void
953 _logsys_log_printf(int level, int subsys,
954  const char *function_name,
955  const char *file_name,
956  int file_line,
957  const char *format, ...)
958 {
959  va_list ap;
960 
961  va_start(ap, format);
962  qb_log_from_external_source_va(function_name, corosync_basename(file_name),
963  format, level, file_line,
964  subsys, ap);
965  va_end(ap);
966 }
967 
968 static void fplay_key_change_notify_fn (
969  int32_t event,
970  const char *key_name,
971  struct icmap_notify_value new_val,
972  struct icmap_notify_value old_val,
973  void *user_data)
974 {
975  if (strcmp(key_name, "runtime.blackbox.dump_flight_data") == 0) {
976  fprintf(stderr,"Writetofile\n");
977  corosync_blackbox_write_to_file ();
978  }
979  if (strcmp(key_name, "runtime.blackbox.dump_state") == 0) {
980  fprintf(stderr,"statefump\n");
982  }
983 }
984 
985 static void corosync_fplay_control_init (void)
986 {
987  icmap_track_t track = NULL;
988 
989  icmap_set_string("runtime.blackbox.dump_flight_data", "no");
990  icmap_set_string("runtime.blackbox.dump_state", "no");
991 
992  icmap_track_add("runtime.blackbox.dump_flight_data",
994  fplay_key_change_notify_fn,
995  NULL, &track);
996  icmap_track_add("runtime.blackbox.dump_state",
998  fplay_key_change_notify_fn,
999  NULL, &track);
1000 }
1001 
1002 static void force_gather_notify_fn(
1003  int32_t event,
1004  const char *key_name,
1005  struct icmap_notify_value new_val,
1006  struct icmap_notify_value old_val,
1007  void *user_data)
1008 {
1009  char *key_val;
1010 
1011  if (icmap_get_string(key_name, &key_val) == CS_OK && strcmp(key_val, "no") == 0)
1012  goto out;
1013 
1014  icmap_set_string("runtime.force_gather", "no");
1015 
1016  if (strcmp(key_name, "runtime.force_gather") == 0) {
1017  log_printf(LOGSYS_LEVEL_ERROR, "Forcing into GATHER state\n");
1019  }
1020 
1021 out:
1022  free(key_val);
1023 }
1024 
1025 static void corosync_force_gather_init (void)
1026 {
1027  icmap_track_t track = NULL;
1028 
1029  icmap_set_string("runtime.force_gather", "no");
1030 
1031  icmap_track_add("runtime.force_gather",
1033  force_gather_notify_fn,
1034  NULL, &track);
1035 }
1036 
1037 /*
1038  * Set RO flag for keys, which ether doesn't make sense to change by user (statistic)
1039  * or which when changed are not reflected by runtime (totem.crypto_cipher, ...).
1040  *
1041  * Also some RO keys cannot be determined in this stage, so they are set later in
1042  * other functions (like nodelist.local_node_pos, ...)
1043  */
1044 static void set_icmap_ro_keys_flag (void)
1045 {
1046  /*
1047  * Set RO flag for all keys of internal configuration and runtime statistics
1048  */
1049  icmap_set_ro_access("internal_configuration.", CS_TRUE, CS_TRUE);
1050  icmap_set_ro_access("runtime.services.", CS_TRUE, CS_TRUE);
1051  icmap_set_ro_access("runtime.config.", CS_TRUE, CS_TRUE);
1052  icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE);
1053  icmap_set_ro_access("uidgid.config.", CS_TRUE, CS_TRUE);
1054  icmap_set_ro_access("system.", CS_TRUE, CS_TRUE);
1055  icmap_set_ro_access("nodelist.", CS_TRUE, CS_TRUE);
1056 
1057  /*
1058  * Set RO flag for constrete keys of configuration which can't be changed
1059  * during runtime
1060  */
1061  icmap_set_ro_access("totem.crypto_cipher", CS_FALSE, CS_TRUE);
1062  icmap_set_ro_access("totem.crypto_hash", CS_FALSE, CS_TRUE);
1063  icmap_set_ro_access("totem.crypto_model", CS_FALSE, CS_TRUE);
1064  icmap_set_ro_access("totem.keyfile", CS_FALSE, CS_TRUE);
1065  icmap_set_ro_access("totem.key", CS_FALSE, CS_TRUE);
1066  icmap_set_ro_access("totem.secauth", CS_FALSE, CS_TRUE);
1067  icmap_set_ro_access("totem.ip_version", CS_FALSE, CS_TRUE);
1068  icmap_set_ro_access("totem.rrp_mode", CS_FALSE, CS_TRUE);
1069  icmap_set_ro_access("totem.transport", CS_FALSE, CS_TRUE);
1070  icmap_set_ro_access("totem.cluster_name", CS_FALSE, CS_TRUE);
1071  icmap_set_ro_access("totem.netmtu", CS_FALSE, CS_TRUE);
1072  icmap_set_ro_access("totem.threads", CS_FALSE, CS_TRUE);
1073  icmap_set_ro_access("totem.version", CS_FALSE, CS_TRUE);
1074  icmap_set_ro_access("totem.nodeid", CS_FALSE, CS_TRUE);
1075  icmap_set_ro_access("totem.clear_node_high_bit", CS_FALSE, CS_TRUE);
1076  icmap_set_ro_access("config.reload_in_progress", CS_FALSE, CS_TRUE);
1077  icmap_set_ro_access("config.totemconfig_reload_in_progress", CS_FALSE, CS_TRUE);
1078 }
1079 
1080 static void main_service_ready (void)
1081 {
1082  int res;
1083 
1084  /*
1085  * This must occur after totempg is initialized because "this_ip" must be set
1086  */
1088  if (res == -1) {
1089  log_printf (LOGSYS_LEVEL_ERROR, "Could not initialize default services");
1091  }
1092  cs_ipcs_init();
1093  corosync_totem_stats_init ();
1094  corosync_fplay_control_init ();
1095  corosync_force_gather_init ();
1096 
1097  sync_init (
1098  corosync_sync_callbacks_retrieve,
1099  corosync_sync_completed);
1100 }
1101 
1102 static enum e_corosync_done corosync_flock (const char *lockfile, pid_t pid)
1103 {
1104  struct flock lock;
1105  enum e_corosync_done err;
1106  char pid_s[17];
1107  int fd_flag;
1108 
1109  err = COROSYNC_DONE_EXIT;
1110 
1111  lockfile_fd = open (lockfile, O_WRONLY | O_CREAT, 0640);
1112  if (lockfile_fd == -1) {
1113  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create lock file.");
1114  return (COROSYNC_DONE_ACQUIRE_LOCK);
1115  }
1116 
1117 retry_fcntl:
1118  lock.l_type = F_WRLCK;
1119  lock.l_start = 0;
1120  lock.l_whence = SEEK_SET;
1121  lock.l_len = 0;
1122  if (fcntl (lockfile_fd, F_SETLK, &lock) == -1) {
1123  switch (errno) {
1124  case EINTR:
1125  goto retry_fcntl;
1126  break;
1127  case EAGAIN:
1128  case EACCES:
1129  log_printf (LOGSYS_LEVEL_ERROR, "Another Corosync instance is already running.");
1131  goto error_close;
1132  break;
1133  default:
1134  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't acquire lock. Error was %s",
1135  strerror(errno));
1137  goto error_close;
1138  break;
1139  }
1140  }
1141 
1142  if (ftruncate (lockfile_fd, 0) == -1) {
1143  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't truncate lock file. Error was %s",
1144  strerror (errno));
1146  goto error_close_unlink;
1147  }
1148 
1149  memset (pid_s, 0, sizeof (pid_s));
1150  snprintf (pid_s, sizeof (pid_s) - 1, "%u\n", pid);
1151 
1152 retry_write:
1153  if (write (lockfile_fd, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1154  if (errno == EINTR) {
1155  goto retry_write;
1156  } else {
1157  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't write pid to lock file. "
1158  "Error was %s", strerror (errno));
1160  goto error_close_unlink;
1161  }
1162  }
1163 
1164  if ((fd_flag = fcntl (lockfile_fd, F_GETFD, 0)) == -1) {
1165  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't get close-on-exec flag from lock file. "
1166  "Error was %s", strerror (errno));
1168  goto error_close_unlink;
1169  }
1170  fd_flag |= FD_CLOEXEC;
1171  if (fcntl (lockfile_fd, F_SETFD, fd_flag) == -1) {
1172  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't set close-on-exec flag to lock file. "
1173  "Error was %s", strerror (errno));
1175  goto error_close_unlink;
1176  }
1177 
1178  return (err);
1179 
1180 error_close_unlink:
1181  unlink (lockfile);
1182 error_close:
1183  close (lockfile_fd);
1184 
1185  return (err);
1186 }
1187 
1188 static int corosync_move_to_root_cgroup(void) {
1189  FILE *f;
1190  int res = -1;
1191  const char *cgroup_task_fname = NULL;
1192 
1193  /*
1194  * /sys/fs/cgroup is hardcoded, because most of Linux distributions are now
1195  * using systemd and systemd uses hardcoded path of cgroup mount point.
1196  *
1197  * This feature is expected to be removed as soon as systemd gets support
1198  * for managing RT configuration.
1199  */
1200  f = fopen("/sys/fs/cgroup/cpu/cpu.rt_runtime_us", "rt");
1201  if (f == NULL) {
1202  /*
1203  * Try cgroup v2
1204  */
1205  f = fopen("/sys/fs/cgroup/cgroup.procs", "rt");
1206  if (f == NULL) {
1207  log_printf(LOG_DEBUG, "cpu.rt_runtime_us or cgroup.procs doesn't exist -> "
1208  "system without cgroup or with disabled CONFIG_RT_GROUP_SCHED");
1209 
1210  res = 0;
1211  goto exit_res;
1212  } else {
1213  log_printf(LOGSYS_LEVEL_DEBUG, "Moving main pid to cgroup v2 root cgroup");
1214 
1215  cgroup_task_fname = "/sys/fs/cgroup/cgroup.procs";
1216  }
1217  } else {
1218  log_printf(LOGSYS_LEVEL_DEBUG, "Moving main pid to cgroup v1 root cgroup");
1219 
1220  cgroup_task_fname = "/sys/fs/cgroup/cpu/tasks";
1221  }
1222  (void)fclose(f);
1223 
1224  f = fopen(cgroup_task_fname, "w");
1225  if (f == NULL) {
1226  log_printf(LOGSYS_LEVEL_WARNING, "Can't open cgroups tasks file for writing");
1227 
1228  goto exit_res;
1229  }
1230 
1231  if (fprintf(f, "%jd\n", (intmax_t)getpid()) <= 0) {
1232  log_printf(LOGSYS_LEVEL_WARNING, "Can't write corosync pid into cgroups tasks file");
1233 
1234  goto close_and_exit_res;
1235  }
1236 
1237 close_and_exit_res:
1238  if (fclose(f) != 0) {
1239  log_printf(LOGSYS_LEVEL_WARNING, "Can't close cgroups tasks file");
1240 
1241  goto exit_res;
1242  }
1243 
1244 exit_res:
1245  return (res);
1246 }
1247 
1248 static void show_version_info_crypto(void)
1249 {
1250  const char *error_string;
1251  const char *list_str;
1252 
1253  if (util_is_valid_knet_crypto_model(NULL, &list_str, 1, "", &error_string) != -1) {
1254  printf("Available crypto models: %s\n", list_str);
1255  } else {
1256  perror(error_string);
1257  }
1258 }
1259 
1260 static void show_version_info_compress(void)
1261 {
1262  const char *error_string;
1263  const char *list_str;
1264 
1265  if (util_is_valid_knet_compress_model(NULL, &list_str, 1, "", &error_string) != -1) {
1266  printf("Available compression models: %s\n", list_str);
1267  } else {
1268  perror(error_string);
1269  }
1270 }
1271 
1272 static void show_version_info(void)
1273 {
1274 
1275  printf ("Corosync Cluster Engine, version '%s'\n", VERSION);
1276  printf ("Copyright (c) 2006-2021 Red Hat, Inc.\n");
1277 
1278  printf ("\nBuilt-in features:" PACKAGE_FEATURES "\n");
1279 
1280  show_version_info_crypto();
1281  show_version_info_compress();
1282 }
1283 
1284 int main (int argc, char **argv, char **envp)
1285 {
1286  const char *error_string;
1287  struct totem_config totem_config;
1288  int res, ch;
1289  int background, sched_rr, prio, testonly;
1290  enum move_to_root_cgroup_mode move_to_root_cgroup;
1291  enum e_corosync_done flock_err;
1292  uint64_t totem_config_warnings;
1294  long int tmpli;
1295  char *ep;
1296  char *tmp_str;
1297  int log_subsys_id_totem;
1298  int silent;
1299 
1300  /* default configuration
1301  */
1302  background = 1;
1303  testonly = 0;
1304 
1305  while ((ch = getopt (argc, argv, "c:ftv")) != EOF) {
1306 
1307  switch (ch) {
1308  case 'c':
1309  res = snprintf(corosync_config_file, sizeof(corosync_config_file), "%s", optarg);
1310  if (res >= sizeof(corosync_config_file)) {
1311  fprintf (stderr, "Config file path too long.\n");
1312  syslog (LOGSYS_LEVEL_ERROR, "Config file path too long.");
1313 
1315  return EXIT_FAILURE;
1316  }
1317  break;
1318  case 'f':
1319  background = 0;
1320  break;
1321  case 't':
1322  testonly = 1;
1323  break;
1324  case 'v':
1325  show_version_info();
1327  return EXIT_SUCCESS;
1328 
1329  break;
1330  default:
1331  fprintf(stderr, \
1332  "usage:\n"\
1333  " -c : Corosync config file path.\n"\
1334  " -f : Start application in foreground.\n"\
1335  " -t : Test configuration and exit.\n"\
1336  " -v : Display version, git revision and some useful information about Corosync and exit.\n");
1338  return EXIT_FAILURE;
1339  }
1340  }
1341 
1342 
1343  /*
1344  * Other signals are registered later via qb_loop_signal_add
1345  */
1346  (void)signal (SIGSEGV, sigsegv_handler);
1347  (void)signal (SIGABRT, sigsegv_handler);
1348 #if MSG_NOSIGNAL != 0
1349  (void)signal (SIGPIPE, SIG_IGN);
1350 #endif
1351 
1352  if (icmap_init() != CS_OK) {
1353  fprintf (stderr, "Corosync Executive couldn't initialize configuration component.\n");
1354  syslog (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize configuration component.");
1356  }
1357  set_icmap_ro_keys_flag();
1358 
1359  /*
1360  * Initialize the corosync_api_v1 definition
1361  */
1362  api = apidef_get ();
1363 
1364  res = coroparse_configparse(icmap_get_global_map(), &error_string);
1365  if (res == -1) {
1366  /*
1367  * Logsys can't log properly at this early stage, and we need to get this message out
1368  *
1369  */
1370  fprintf (stderr, "%s\n", error_string);
1371  syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
1373  }
1374 
1375  if (stats_map_init(api) != CS_OK) {
1376  fprintf (stderr, "Corosync Executive couldn't initialize statistics component.\n");
1377  syslog (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize statistics component.");
1379  }
1380 
1381  res = corosync_log_config_read (&error_string);
1382  if (res == -1) {
1383  /*
1384  * if we are here, we _must_ flush the logsys queue
1385  * and try to inform that we couldn't read the config.
1386  * this is a desperate attempt before certain death
1387  * and there is no guarantee that we can print to stderr
1388  * nor that logsys is sending the messages where we expect.
1389  */
1390  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1391  fprintf(stderr, "%s", error_string);
1392  syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
1394  }
1395 
1396  if (!testonly) {
1397  log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Cluster Engine %s starting up", VERSION);
1398  log_printf (LOGSYS_LEVEL_INFO, "Corosync built-in features:" PACKAGE_FEATURES "");
1399  }
1400 
1401  /*
1402  * Create totem logsys subsys before totem_config_read so log functions can be used
1403  */
1404  log_subsys_id_totem = _logsys_subsys_create("TOTEM", "totem,"
1405  "totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
1406  "totempg.c,totemudp.c,totemudpu.c,totemnet.c,totemknet.c");
1407 
1408  res = chdir(get_state_dir());
1409  if (res == -1) {
1410  log_printf (LOGSYS_LEVEL_ERROR, "Cannot chdir to state directory %s. %s", get_state_dir(), strerror(errno));
1412  }
1413 
1414  res = totem_config_read (&totem_config, &error_string, &totem_config_warnings);
1415  if (res == -1) {
1416  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1418  }
1419 
1420  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_IGNORED) {
1421  log_printf (LOGSYS_LEVEL_WARNING, "member section is used together with nodelist. Members ignored.");
1422  }
1423 
1424  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED) {
1425  log_printf (LOGSYS_LEVEL_WARNING, "member section is deprecated.");
1426  }
1427 
1428  if (totem_config_warnings & TOTEM_CONFIG_WARNING_TOTEM_NODEID_SET) {
1429  log_printf (LOGSYS_LEVEL_WARNING, "nodeid in totem section is deprecated and ignored. "
1430  "Nodelist (or autogenerated) nodeid is going to be used.");
1431  }
1432 
1433  if (totem_config_warnings & TOTEM_CONFIG_BINDNETADDR_NODELIST_SET) {
1434  log_printf (LOGSYS_LEVEL_WARNING, "interface section bindnetaddr is used together with nodelist. "
1435  "Nodelist one is going to be used.");
1436  }
1437 
1438  if (totem_config_warnings != 0) {
1439  log_printf (LOGSYS_LEVEL_WARNING, "Please migrate config file to nodelist.");
1440  }
1441 
1442  res = totem_config_validate (&totem_config, &error_string);
1443  if (res == -1) {
1444  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1446  }
1447 
1448  if (testonly) {
1450  }
1451 
1452 
1453  move_to_root_cgroup = MOVE_TO_ROOT_CGROUP_MODE_AUTO;
1454  if (icmap_get_string("system.move_to_root_cgroup", &tmp_str) == CS_OK) {
1455  /*
1456  * Validity of move_to_root_cgroup values checked in coroparse.c
1457  */
1458  if (strcmp(tmp_str, "yes") == 0) {
1459  move_to_root_cgroup = MOVE_TO_ROOT_CGROUP_MODE_ON;
1460  } else if (strcmp(tmp_str, "no") == 0) {
1461  move_to_root_cgroup = MOVE_TO_ROOT_CGROUP_MODE_OFF;
1462  }
1463  free(tmp_str);
1464  }
1465 
1466 
1467  sched_rr = 1;
1468  if (icmap_get_string("system.sched_rr", &tmp_str) == CS_OK) {
1469  if (strcmp(tmp_str, "yes") != 0) {
1470  sched_rr = 0;
1471  }
1472  free(tmp_str);
1473  }
1474 
1475  prio = 0;
1476  if (icmap_get_string("system.priority", &tmp_str) == CS_OK) {
1477  if (strcmp(tmp_str, "max") == 0) {
1478  prio = INT_MIN;
1479  } else if (strcmp(tmp_str, "min") == 0) {
1480  prio = INT_MAX;
1481  } else {
1482  errno = 0;
1483 
1484  tmpli = strtol(tmp_str, &ep, 10);
1485  if (errno != 0 || *ep != '\0' || tmpli > INT_MAX || tmpli < INT_MIN) {
1486  log_printf (LOGSYS_LEVEL_ERROR, "Priority value %s is invalid", tmp_str);
1488  }
1489 
1490  prio = tmpli;
1491  }
1492 
1493  free(tmp_str);
1494  }
1495 
1496  if (move_to_root_cgroup == MOVE_TO_ROOT_CGROUP_MODE_ON) {
1497  /*
1498  * Try to move corosync into root cpu cgroup. Failure is not fatal and
1499  * error is deliberately ignored.
1500  */
1501  (void)corosync_move_to_root_cgroup();
1502  }
1503 
1504  /*
1505  * Set round robin realtime scheduling with priority 99
1506  */
1507  if (sched_rr) {
1508  silent = (move_to_root_cgroup == MOVE_TO_ROOT_CGROUP_MODE_AUTO);
1509  res = corosync_set_rr_scheduler (silent);
1510 
1511  if (res == -1 && move_to_root_cgroup == MOVE_TO_ROOT_CGROUP_MODE_AUTO) {
1512  /*
1513  * Try to move process to root cgroup and try set priority again
1514  */
1515  (void)corosync_move_to_root_cgroup();
1516 
1517  res = corosync_set_rr_scheduler (0);
1518  }
1519 
1520  if (res != 0) {
1521  prio = INT_MIN;
1522  } else {
1523  prio = 0;
1524  }
1525  }
1526 
1527  if (prio != 0) {
1528  if (setpriority(PRIO_PGRP, 0, prio) != 0) {
1530  "Could not set priority %d", prio);
1531  }
1532  }
1533 
1534  totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load;
1535  totem_config.totem_memb_ring_id_store = corosync_ring_id_store;
1536 
1539 
1547 
1549 
1550  /*
1551  * Now we are fully initialized.
1552  */
1553  if (background) {
1555 
1556  corosync_tty_detach ();
1557 
1559 
1560  log_printf (LOGSYS_LEVEL_DEBUG, "Corosync TTY detached");
1561  }
1562 
1563  /*
1564  * Lock all memory to avoid page faults which may interrupt
1565  * application healthchecking
1566  */
1567  corosync_mlockall ();
1568 
1569  corosync_poll_handle = qb_loop_create ();
1570 
1573  timer_function_scheduler_timeout (&scheduler_pause_timeout_data);
1574 
1575  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1576  SIGUSR2, NULL, sig_diag_handler, NULL);
1577  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1578  SIGINT, NULL, sig_exit_handler, NULL);
1579  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1580  SIGQUIT, NULL, sig_exit_handler, NULL);
1581  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1582  SIGTERM, NULL, sig_exit_handler, NULL);
1583 
1584  if (logsys_thread_start() != 0) {
1585  log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize log thread");
1587  }
1588 
1589  if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) != COROSYNC_DONE_EXIT) {
1590  corosync_exit_error (flock_err);
1591  }
1592 
1593  /*
1594  * if totempg_initialize doesn't have root priveleges, it cannot
1595  * bind to a specific interface. This only matters if
1596  * there is more then one interface in a system, so
1597  * in this case, only a warning is printed
1598  */
1599  /*
1600  * Join multicast group and setup delivery
1601  * and configuration change functions
1602  */
1603  if (totempg_initialize (
1604  corosync_poll_handle,
1605  &totem_config) != 0) {
1606 
1607  log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize TOTEM layer");
1609  }
1610 
1612  main_service_ready);
1613 
1615  &corosync_group_handle,
1616  deliver_fn,
1617  confchg_fn);
1618 
1620  corosync_group_handle,
1621  &corosync_group,
1622  1);
1623 
1624  /*
1625  * Drop root privleges to user 'corosync'
1626  * TODO: Don't really need full root capabilities;
1627  * needed capabilities are:
1628  * CAP_NET_RAW (bindtodevice)
1629  * CAP_SYS_NICE (setscheduler)
1630  * CAP_IPC_LOCK (mlockall)
1631  */
1632  priv_drop ();
1633 
1634  schedwrk_init (
1635  serialize_lock,
1636  serialize_unlock);
1637 
1638  /*
1639  * Start main processing loop
1640  */
1641  qb_loop_run (corosync_poll_handle);
1642 
1643  /*
1644  * Exit was requested
1645  */
1646  totempg_finalize ();
1647 
1648  /*
1649  * free the loop resources
1650  */
1651  qb_loop_destroy (corosync_poll_handle);
1652 
1653  /*
1654  * free up the icmap
1655  */
1656 
1657  /*
1658  * Remove pid lock file
1659  */
1660  close (lockfile_fd);
1661  unlink (corosync_lock_file);
1662 
1664 
1665  return EXIT_SUCCESS;
1666 }
struct corosync_api_v1 * apidef_get(void)
Definition: apidef.c:146
#define VERSION
Definition: config.h:448
#define PACKAGE_FEATURES
Definition: config.h:388
#define LOCALSTATEDIR
Definition: config.h:373
#define COROSYSCONFDIR
Definition: config.h:8
totem_configuration_type
The totem_configuration_type enum.
Definition: coroapi.h:132
@ TOTEM_CONFIGURATION_REGULAR
Definition: coroapi.h:133
@ TOTEM_CONFIGURATION_TRANSITIONAL
Definition: coroapi.h:134
@ CS_LIB_ALLOW_INQUORATE
Definition: coroapi.h:164
qb_loop_timer_handle corosync_timer_handle_t
corosync_timer_handle_t
Definition: coroapi.h:74
unsigned int nodeid
Definition: coroapi.h:0
@ CS_LIB_FLOW_CONTROL_NOT_REQUIRED
Definition: coroapi.h:153
#define SERVICES_COUNT_MAX
Definition: coroapi.h:462
#define MILLI_2_NANO_SECONDS
Definition: coroapi.h:105
int coroparse_configparse(icmap_map_t config_map, const char **error_string)
Definition: coroparse.c:258
#define CS_PRI_RING_ID_SEQ
Definition: corotypes.h:61
#define CS_FALSE
Definition: corotypes.h:53
#define CS_TRUE
Definition: corotypes.h:54
@ CS_OK
Definition: corotypes.h:99
int corosync_quorum_is_quorate(void)
Definition: exec/quorum.c:65
#define corosync_exit_error(err)
Definition: exec/util.h:72
e_corosync_done
Definition: exec/util.h:47
@ COROSYNC_DONE_INIT_SERVICES
Definition: exec/util.h:54
@ COROSYNC_DONE_STATS
Definition: exec/util.h:62
@ COROSYNC_DONE_ICMAP
Definition: exec/util.h:53
@ COROSYNC_DONE_ALREADY_RUNNING
Definition: exec/util.h:58
@ COROSYNC_DONE_FORK
Definition: exec/util.h:49
@ COROSYNC_DONE_STORE_RINGID
Definition: exec/util.h:61
@ COROSYNC_DONE_EXIT
Definition: exec/util.h:48
@ COROSYNC_DONE_LOGSETUP
Definition: exec/util.h:52
@ COROSYNC_DONE_FATAL_ERR
Definition: exec/util.h:55
@ COROSYNC_DONE_LOGCONFIGREAD
Definition: exec/util.h:50
@ COROSYNC_DONE_STD_TO_NULL_REDIR
Definition: exec/util.h:59
@ COROSYNC_DONE_MAINCONFIGREAD
Definition: exec/util.h:51
@ COROSYNC_DONE_ACQUIRE_LOCK
Definition: exec/util.h:57
@ COROSYNC_DONE_DIR_NOT_PRESENT
Definition: exec/util.h:56
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
cs_error_t icmap_set_ro_access(const char *key_name, int prefix, int ro_access)
Set read-only access for given key (key_name) or prefix, If prefix is set.
Definition: icmap.c:1225
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1159
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition: icmap.c:627
cs_error_t icmap_inc(const char *key_name)
Increase stored value by one.
Definition: icmap.c:1049
cs_error_t icmap_fast_inc(const char *key_name)
Increase stored value by one.
Definition: icmap.c:1069
icmap_map_t icmap_get_global_map(void)
Return global icmap.
Definition: icmap.c:264
cs_error_t icmap_get(const char *key_name, void *value, size_t *value_len, icmap_value_types_t *type)
Retrieve value of key key_name and store it in user preallocated value pointer.
Definition: icmap.c:725
cs_error_t icmap_init(void)
Initialize global icmap.
Definition: icmap.c:208
#define ICMAP_KEYNAME_MAXLEN
Maximum length of key in icmap.
Definition: icmap.h:48
void icmap_fini(void)
Finalize global icmap.
Definition: icmap.c:247
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
Definition: icmap.c:597
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:856
void cs_ipcs_init(void)
Definition: ipc_glue.c:818
void cs_ipcs_sync_state_changed(int32_t sync_in_process)
Definition: ipc_glue.c:664
int32_t cs_ipcs_q_level_get(void)
Definition: ipc_glue.c:601
void cs_ipc_allow_connections(int32_t allow)
Definition: ipc_glue.c:159
int corosync_log_config_read(cmap_handle_t cmap_h, const char *default_logfile, const char **error_string)
Definition: logconfig.c:715
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
#define log_printf(level, format, args...)
Definition: logsys.h:323
void logsys_blackbox_prefork(void)
Definition: logsys.c:878
void logsys_system_fini(void)
logsys_system_fini
Definition: logsys.c:286
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
#define LOGSYS_MODE_OUTPUT_STDERR
Definition: logsys.h:61
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:74
void logsys_config_apply(void)
logsys_config_apply
Definition: logsys.c:790
#define LOGSYS_PERROR(err_num, level, fmt, args...)
The LOGSYS_PERROR macro.
Definition: logsys.h:317
int logsys_thread_start(void)
logsys_thread_start
Definition: logsys.c:842
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
void logsys_blackbox_postfork(void)
Definition: logsys.c:884
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
#define LOGSYS_MODE_OUTPUT_SYSLOG
Definition: logsys.h:62
#define LOGSYS_LEVEL_TRACE
Definition: logsys.h:77
void corosync_sending_allowed_release(void *sending_allowed_private_data)
Definition: main.c:785
LOGSYS_DECLARE_SYSTEM("corosync", LOGSYS_MODE_OUTPUT_STDERR|LOGSYS_MODE_OUTPUT_SYSLOG, LOG_DAEMON, LOG_EMERG)
void corosync_state_dump(void)
Definition: main.c:201
const char * corosync_get_config_file(void)
Definition: main.c:212
int cs_poll_dispatch_add(qb_loop_t *handle, int fd, int events, void *data, int(*dispatch_fn)(int fd, int revents, void *data))
Definition: main.c:183
move_to_root_cgroup_mode
Definition: main.c:172
@ MOVE_TO_ROOT_CGROUP_MODE_ON
Definition: main.c:174
@ MOVE_TO_ROOT_CGROUP_MODE_OFF
Definition: main.c:173
@ MOVE_TO_ROOT_CGROUP_MODE_AUTO
Definition: main.c:175
#define RLIMIT_MEMLOCK
struct sched_param global_sched_param
Definition: main.c:162
void message_source_set(mar_message_source_t *source, void *conn)
Definition: main.c:807
int main(int argc, char **argv, char **envp)
Definition: main.c:1284
int main_mcast(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
Definition: main.c:627
void corosync_recheck_the_q_level(void *data)
Definition: main.c:723
int message_source_is_local(const mar_message_source_t *source)
Definition: main.c:796
qb_loop_t * cs_poll_handle_get(void)
Definition: main.c:178
LOGSYS_DECLARE_SUBSYS("MAIN")
int corosync_sending_allowed(unsigned int service, unsigned int id, const void *msg, void *sending_allowed_private_data)
Definition: main.c:737
int cs_poll_dispatch_delete(qb_loop_t *handle, int fd)
Definition: main.c:196
void corosync_shutdown_request(void)
Definition: main.c:258
pthread_mutex_t lock
Definition: sam.c:131
void * user_data
Definition: sam.c:127
void schedwrk_init(void(*serialize_lock_fn)(void), void(*serialize_unlock_fn)(void))
Definition: schedwrk.c:83
unsigned int corosync_service_defaults_link_and_init(struct corosync_api_v1 *corosync_api)
Load all of the default services.
Definition: service.c:339
void corosync_service_unlink_all(struct corosync_api_v1 *api, void(*unlink_all_complete)(void))
Unlink and exit all corosync services.
Definition: service.c:394
const char * service_stats_rx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:112
struct corosync_service_engine * corosync_service[SERVICES_COUNT_MAX]
Definition: service.c:110
const char * service_stats_tx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:113
void stats_add_schedmiss_event(uint64_t timestamp, float delay)
Definition: stats.c:447
void stats_trigger_trackers()
Definition: stats.c:561
cs_error_t stats_map_init(const struct corosync_api_v1 *corosync_api)
Definition: stats.c:289
The corosync_api_v1 struct.
Definition: coroapi.h:225
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
Definition: coroapi.h:229
void(* timer_delete)(corosync_timer_handle_t timer_handle)
Definition: coroapi.h:241
const char *(* totem_ifaces_print)(unsigned int nodeid)
Definition: coroapi.h:290
void *(* totem_get_stats)(void)
Definition: coroapi.h:435
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
Definition: coroapi.h:476
void(* exec_endian_convert_fn)(void *msg)
Definition: coroapi.h:477
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:515
const char * name
Definition: coroapi.h:491
void(* sync_activate)(void)
Definition: coroapi.h:522
void(* confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:509
void(* sync_abort)(void)
Definition: coroapi.h:523
struct corosync_exec_handler * exec_engine
Definition: coroapi.h:506
void(* exec_dump_fn)(void)
Definition: coroapi.h:501
int(* sync_process)(void)
Definition: coroapi.h:521
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
The mar_message_source_t struct.
Definition: coroapi.h:50
The memb_ring_id struct.
Definition: coroapi.h:122
unsigned long long seq
Definition: coroapi.h:124
unsigned int rep
Definition: totem.h:150
qb_loop_timer_handle handle
Definition: main.c:819
unsigned long long max_tv_diff
Definition: main.c:821
struct totem_config * totem_config
Definition: main.c:818
unsigned long long tv_prev
Definition: main.c:820
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.h:39
const char * name
Definition: sync.h:48
int(* sync_process)(void)
Definition: sync.h:45
void(* sync_activate)(void)
Definition: sync.h:46
void(* sync_abort)(void)
Definition: sync.h:47
unsigned int token_timeout
Definition: totem.h:181
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:207
void(* totem_memb_ring_id_create_or_load)(struct memb_ring_id *memb_ring_id, unsigned int nodeid)
Definition: totem.h:249
void(* totem_memb_ring_id_store)(const struct memb_ring_id *memb_ring_id, unsigned int nodeid)
Definition: totem.h:253
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:101
void(*) in log_level_security)
Definition: totem.h:108
const void * group
Definition: totempg.h:56
totemsrp_stats_t * srp
Definition: totemstats.h:96
uint32_t continuous_gather
Definition: totemstats.h:78
uint32_t avg_backlog_calc
Definition: totemstats.h:85
uint32_t continuous_sendmsg_failures
Definition: totemstats.h:79
totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX]
Definition: totemstats.h:90
uint8_t firewall_enabled_or_nic_failure
Definition: totemstats.h:82
uint32_t avg_token_workload
Definition: totemstats.h:84
uint32_t mtt_rx_token
Definition: totemstats.h:83
uint64_t time_since_token_last_received
Definition: totemstats.h:80
#define swab32(x)
The swab32 macro.
Definition: swab.h:51
int sync_init(int(*sync_callbacks_retrieve)(int service_id, struct sync_callbacks *callbacks), void(*synchronization_completed)(void))
Definition: sync.c:158
void sync_abort(void)
Definition: sync.c:535
void sync_save_transitional(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:524
void sync_start(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:512
typedef __attribute__
#define MAX_NO_CONT_SENDMSG_FAILURES
Definition: totem.h:80
#define MAX_NO_CONT_GATHER
Maximum number of continuous gather states.
Definition: totem.h:76
int totem_config_read(struct totem_config *totem_config, const char **error_string, uint64_t *warnings)
Definition: totemconfig.c:1780
int totem_config_validate(struct totem_config *totem_config, const char **error_string)
Definition: totemconfig.c:1997
#define TOTEM_CONFIG_BINDNETADDR_NODELIST_SET
Definition: totemconfig.h:48
#define TOTEM_CONFIG_WARNING_TOTEM_NODEID_SET
Definition: totemconfig.h:47
#define TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
Definition: totemconfig.h:45
#define TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
Definition: totemconfig.h:46
Totem Single Ring Protocol.
unsigned int totempg_my_nodeid_get(void)
Definition: totempg.c:1532
void totempg_check_q_level(void *instance)
Definition: totempg.c:1294
int totempg_groups_mcast_joined(void *instance, const struct iovec *iovec, unsigned int iov_len, int guarantee)
Definition: totempg.c:1232
void totempg_trans_ack(void)
Definition: totempg.c:1591
void totempg_force_gather(void)
Definition: totempg.c:1596
int totempg_initialize(qb_loop_t *poll_handle, struct totem_config *totem_config)
Initialize the totem process groups abstraction.
Definition: totempg.c:802
int totempg_groups_joined_release(int msg_count)
Definition: totempg.c:1346
void totempg_service_ready_register(void(*totem_service_ready)(void))
Definition: totempg.c:1541
int totempg_groups_joined_reserve(void *instance, const struct iovec *iovec, unsigned int iov_len)
Definition: totempg.c:1302
int totempg_groups_join(void *instance, const struct totempg_group *groups, size_t group_cnt)
Definition: totempg.c:1182
void totempg_finalize(void)
Definition: totempg.c:855
@ TOTEM_Q_LEVEL_CRITICAL
Definition: totempg.h:184
int totempg_groups_initialize(void **instance, void(*deliver_fn)(unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void(*confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id))
Initialize a groups instance.
Definition: totempg.c:1134
int guarantee
Definition: totemsrp.c:6
struct memb_ring_id ring_id
Definition: totemsrp.c:4
struct totem_message_header header
Definition: totemsrp.c:0
Totem Single Ring Protocol.
#define TOTEM_TOKEN_STATS_MAX
Definition: totemstats.h:89
int util_is_valid_knet_crypto_model(const char *val, const char **list_str, int machine_parseable_str, const char *error_string_prefix, const char **error_string)
Definition: util.c:213
int util_is_valid_knet_compress_model(const char *val, const char **list_str, int machine_parseable_str, const char *error_string_prefix, const char **error_string)
Definition: util.c:279
const char * get_state_dir(void)
Definition: util.c:174