Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame^] | 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
| 2 | From: Benjamin Marzinski <bmarzins@redhat.com> |
| 3 | Date: Thu, 31 May 2018 17:47:11 -0500 |
| 4 | Subject: [PATCH] mpathpersist: add all_tg_pt option |
| 5 | |
| 6 | Some arrays, such as the EMC VNX, don't follow the scsi persistent |
| 7 | reservations spec in making key registrations per I_T NEXUS. Instead, |
| 8 | the registration is shared by all target ports connected to a given |
| 9 | host. This causes mpathpersist to fail whenever it tries to register a |
| 10 | key, since it will receive a registration conflict on some of the paths. |
| 11 | |
| 12 | To deal with this, mpathpersist needs to track the hosts that it has |
| 13 | done a registration on, and only register once per host. The new |
| 14 | "all_tg_pt" multipath.conf option is used to set which arrays need this |
| 15 | feature. I currently don't know if all EMC VNX arrays handle persistent |
| 16 | reservations like this, or if it is configurable. A future patch will |
| 17 | update the VNX built-in config, if this is indeed their default (or |
| 18 | only) setting. |
| 19 | |
| 20 | Multipathd doesn't need to worry about this. It is often the case that |
| 21 | when a path device comes back, it will still have the keys registered to |
| 22 | it. Because of this, multipathd uses register-and-ignore, which means |
| 23 | that it won't cause an error if the registration has already happened |
| 24 | down a different target port. |
| 25 | |
| 26 | Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com> |
| 27 | --- |
| 28 | libmpathpersist/mpath_persist.c | 28 ++++++++++++++++++++++------ |
| 29 | libmultipath/config.c | 2 ++ |
| 30 | libmultipath/config.h | 2 ++ |
| 31 | libmultipath/defaults.h | 1 + |
| 32 | libmultipath/dict.c | 10 ++++++++++ |
| 33 | libmultipath/propsel.c | 15 +++++++++++++++ |
| 34 | libmultipath/propsel.h | 1 + |
| 35 | libmultipath/structs.h | 7 +++++++ |
| 36 | multipath/multipath.conf.5 | 11 +++++++++++ |
| 37 | 9 files changed, 71 insertions(+), 6 deletions(-) |
| 38 | |
| 39 | diff --git a/libmpathpersist/mpath_persist.c b/libmpathpersist/mpath_persist.c |
| 40 | index 907a17c..ca91c55 100644 |
| 41 | --- a/libmpathpersist/mpath_persist.c |
| 42 | +++ b/libmpathpersist/mpath_persist.c |
| 43 | @@ -335,6 +335,7 @@ int mpath_persistent_reserve_out ( int fd, int rq_servact, int rq_scope, |
| 44 | |
| 45 | conf = get_multipath_config(); |
| 46 | select_reservation_key(conf, mpp); |
| 47 | + select_all_tg_pt(conf, mpp); |
| 48 | put_multipath_config(conf); |
| 49 | |
| 50 | memcpy(&prkey, paramp->sa_key, 8); |
| 51 | @@ -456,7 +457,7 @@ int mpath_prout_reg(struct multipath *mpp,int rq_servact, int rq_scope, |
| 52 | unsigned int rq_type, struct prout_param_descriptor * paramp, int noisy) |
| 53 | { |
| 54 | |
| 55 | - int i, j; |
| 56 | + int i, j, k; |
| 57 | struct pathgroup *pgp = NULL; |
| 58 | struct path *pp = NULL; |
| 59 | int rollback = 0; |
| 60 | @@ -481,11 +482,13 @@ int mpath_prout_reg(struct multipath *mpp,int rq_servact, int rq_scope, |
| 61 | } |
| 62 | |
| 63 | struct threadinfo thread[active_pathcount]; |
| 64 | + int hosts[active_pathcount]; |
| 65 | |
| 66 | memset(thread, 0, sizeof(thread)); |
| 67 | |
| 68 | /* init thread parameter */ |
| 69 | for (i =0; i< active_pathcount; i++){ |
| 70 | + hosts[i] = -1; |
| 71 | thread[i].param.rq_servact = rq_servact; |
| 72 | thread[i].param.rq_scope = rq_scope; |
| 73 | thread[i].param.rq_type = rq_type; |
| 74 | @@ -514,6 +517,17 @@ int mpath_prout_reg(struct multipath *mpp,int rq_servact, int rq_scope, |
| 75 | condlog (1, "%s: %s path not up. Skip.", mpp->wwid, pp->dev); |
| 76 | continue; |
| 77 | } |
| 78 | + if (mpp->all_tg_pt == ALL_TG_PT_ON && |
| 79 | + pp->sg_id.host_no != -1) { |
| 80 | + for (k = 0; k < count; k++) { |
| 81 | + if (pp->sg_id.host_no == hosts[k]) { |
| 82 | + condlog(3, "%s: %s host %d matches skip.", pp->wwid, pp->dev, pp->sg_id.host_no); |
| 83 | + break; |
| 84 | + } |
| 85 | + } |
| 86 | + if (k < count) |
| 87 | + continue; |
| 88 | + } |
| 89 | strncpy(thread[count].param.dev, pp->dev, |
| 90 | FILE_NAME_SIZE - 1); |
| 91 | |
| 92 | @@ -531,10 +545,12 @@ int mpath_prout_reg(struct multipath *mpp,int rq_servact, int rq_scope, |
| 93 | condlog (0, "%s: failed to create thread %d", mpp->wwid, rc); |
| 94 | thread[count].param.status = MPATH_PR_THREAD_ERROR; |
| 95 | } |
| 96 | + else |
| 97 | + hosts[count] = pp->sg_id.host_no; |
| 98 | count = count + 1; |
| 99 | } |
| 100 | } |
| 101 | - for( i=0; i < active_pathcount ; i++){ |
| 102 | + for( i=0; i < count ; i++){ |
| 103 | if (thread[i].param.status != MPATH_PR_THREAD_ERROR) { |
| 104 | rc = pthread_join(thread[i].id, NULL); |
| 105 | if (rc){ |
| 106 | @@ -557,7 +573,7 @@ int mpath_prout_reg(struct multipath *mpp,int rq_servact, int rq_scope, |
| 107 | } |
| 108 | if (rollback && ((rq_servact == MPATH_PROUT_REG_SA) && sa_key != 0 )){ |
| 109 | condlog (3, "%s: ERROR: initiating pr out rollback", mpp->wwid); |
| 110 | - for( i=0 ; i < active_pathcount ; i++){ |
| 111 | + for( i=0 ; i < count ; i++){ |
| 112 | if(thread[i].param.status == MPATH_PR_SUCCESS) { |
| 113 | memcpy(&thread[i].param.paramp->key, &thread[i].param.paramp->sa_key, 8); |
| 114 | memset(&thread[i].param.paramp->sa_key, 0, 8); |
| 115 | @@ -571,7 +587,7 @@ int mpath_prout_reg(struct multipath *mpp,int rq_servact, int rq_scope, |
| 116 | } else |
| 117 | thread[i].param.status = MPATH_PR_SKIP; |
| 118 | } |
| 119 | - for(i=0; i < active_pathcount ; i++){ |
| 120 | + for(i=0; i < count ; i++){ |
| 121 | if (thread[i].param.status != MPATH_PR_SKIP && |
| 122 | thread[i].param.status != MPATH_PR_THREAD_ERROR) { |
| 123 | rc = pthread_join(thread[i].id, NULL); |
| 124 | @@ -720,7 +736,7 @@ int mpath_prout_rel(struct multipath *mpp,int rq_servact, int rq_scope, |
| 125 | } |
| 126 | } |
| 127 | pthread_attr_destroy (&attr); |
| 128 | - for (i = 0; i < active_pathcount; i++){ |
| 129 | + for (i = 0; i < count; i++){ |
| 130 | if (thread[i].param.status != MPATH_PR_THREAD_ERROR) { |
| 131 | rc = pthread_join (thread[i].id, NULL); |
| 132 | if (rc){ |
| 133 | @@ -729,7 +745,7 @@ int mpath_prout_rel(struct multipath *mpp,int rq_servact, int rq_scope, |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | - for (i = 0; i < active_pathcount; i++){ |
| 138 | + for (i = 0; i < count; i++){ |
| 139 | /* check thread status here and return the status */ |
| 140 | |
| 141 | if (thread[i].param.status == MPATH_PR_RESERV_CONFLICT) |
| 142 | diff --git a/libmultipath/config.c b/libmultipath/config.c |
| 143 | index 085a3e1..5872927 100644 |
| 144 | --- a/libmultipath/config.c |
| 145 | +++ b/libmultipath/config.c |
| 146 | @@ -352,6 +352,7 @@ merge_hwe (struct hwentry * dst, struct hwentry * src) |
| 147 | merge_num(skip_kpartx); |
| 148 | merge_num(max_sectors_kb); |
| 149 | merge_num(ghost_delay); |
| 150 | + merge_num(all_tg_pt); |
| 151 | |
| 152 | snprintf(id, sizeof(id), "%s/%s", dst->vendor, dst->product); |
| 153 | reconcile_features_with_options(id, &dst->features, |
| 154 | @@ -622,6 +623,7 @@ load_config (char * file) |
| 155 | conf->disable_changed_wwids = DEFAULT_DISABLE_CHANGED_WWIDS; |
| 156 | conf->remove_retries = 0; |
| 157 | conf->ghost_delay = DEFAULT_GHOST_DELAY; |
| 158 | + conf->all_tg_pt = DEFAULT_ALL_TG_PT; |
| 159 | |
| 160 | /* |
| 161 | * preload default hwtable |
| 162 | diff --git a/libmultipath/config.h b/libmultipath/config.h |
| 163 | index 6e69a37..1bf708a 100644 |
| 164 | --- a/libmultipath/config.h |
| 165 | +++ b/libmultipath/config.h |
| 166 | @@ -82,6 +82,7 @@ struct hwentry { |
| 167 | int skip_kpartx; |
| 168 | int max_sectors_kb; |
| 169 | int ghost_delay; |
| 170 | + int all_tg_pt; |
| 171 | char * bl_product; |
| 172 | }; |
| 173 | |
| 174 | @@ -194,6 +195,7 @@ struct config { |
| 175 | char * partition_delim; |
| 176 | char * config_dir; |
| 177 | int prkey_source; |
| 178 | + int all_tg_pt; |
| 179 | struct be64 reservation_key; |
| 180 | |
| 181 | vector keywords; |
| 182 | diff --git a/libmultipath/defaults.h b/libmultipath/defaults.h |
| 183 | index d7b87b4..f076b4b 100644 |
| 184 | --- a/libmultipath/defaults.h |
| 185 | +++ b/libmultipath/defaults.h |
| 186 | @@ -43,6 +43,7 @@ |
| 187 | #define DEFAULT_GHOST_DELAY GHOST_DELAY_OFF |
| 188 | #define DEFAULT_FIND_MULTIPATHS_TIMEOUT -10 |
| 189 | #define DEFAULT_UNKNOWN_FIND_MULTIPATHS_TIMEOUT 1 |
| 190 | +#define DEFAULT_ALL_TG_PT ALL_TG_PT_OFF |
| 191 | |
| 192 | #define DEFAULT_CHECKINT 5 |
| 193 | #define MAX_CHECKINT(a) (a << 2) |
| 194 | diff --git a/libmultipath/dict.c b/libmultipath/dict.c |
| 195 | index 3e7c5d6..2557b8a 100644 |
| 196 | --- a/libmultipath/dict.c |
| 197 | +++ b/libmultipath/dict.c |
| 198 | @@ -1178,6 +1178,13 @@ declare_hw_snprint(ghost_delay, print_off_int_undef) |
| 199 | declare_mp_handler(ghost_delay, set_off_int_undef) |
| 200 | declare_mp_snprint(ghost_delay, print_off_int_undef) |
| 201 | |
| 202 | +declare_def_handler(all_tg_pt, set_yes_no_undef) |
| 203 | +declare_def_snprint_defint(all_tg_pt, print_yes_no_undef, DEFAULT_ALL_TG_PT) |
| 204 | +declare_ovr_handler(all_tg_pt, set_yes_no_undef) |
| 205 | +declare_ovr_snprint(all_tg_pt, print_yes_no_undef) |
| 206 | +declare_hw_handler(all_tg_pt, set_yes_no_undef) |
| 207 | +declare_hw_snprint(all_tg_pt, print_yes_no_undef) |
| 208 | + |
| 209 | |
| 210 | static int |
| 211 | def_uxsock_timeout_handler(struct config *conf, vector strvec) |
| 212 | @@ -1509,6 +1516,7 @@ init_keywords(vector keywords) |
| 213 | install_keyword("prkeys_file", &def_prkeys_file_handler, &snprint_def_prkeys_file); |
| 214 | install_keyword("log_checker_err", &def_log_checker_err_handler, &snprint_def_log_checker_err); |
| 215 | install_keyword("reservation_key", &def_reservation_key_handler, &snprint_def_reservation_key); |
| 216 | + install_keyword("all_tg_pt", &def_all_tg_pt_handler, &snprint_def_all_tg_pt); |
| 217 | install_keyword("retain_attached_hw_handler", &def_retain_hwhandler_handler, &snprint_def_retain_hwhandler); |
| 218 | install_keyword("detect_prio", &def_detect_prio_handler, &snprint_def_detect_prio); |
| 219 | install_keyword("detect_checker", &def_detect_checker_handler, &snprint_def_detect_checker); |
| 220 | @@ -1618,6 +1626,7 @@ init_keywords(vector keywords) |
| 221 | install_keyword("skip_kpartx", &hw_skip_kpartx_handler, &snprint_hw_skip_kpartx); |
| 222 | install_keyword("max_sectors_kb", &hw_max_sectors_kb_handler, &snprint_hw_max_sectors_kb); |
| 223 | install_keyword("ghost_delay", &hw_ghost_delay_handler, &snprint_hw_ghost_delay); |
| 224 | + install_keyword("all_tg_pt", &hw_all_tg_pt_handler, &snprint_hw_all_tg_pt); |
| 225 | install_sublevel_end(); |
| 226 | |
| 227 | install_keyword_root("overrides", &overrides_handler); |
| 228 | @@ -1654,6 +1663,7 @@ init_keywords(vector keywords) |
| 229 | install_keyword("skip_kpartx", &ovr_skip_kpartx_handler, &snprint_ovr_skip_kpartx); |
| 230 | install_keyword("max_sectors_kb", &ovr_max_sectors_kb_handler, &snprint_ovr_max_sectors_kb); |
| 231 | install_keyword("ghost_delay", &ovr_ghost_delay_handler, &snprint_ovr_ghost_delay); |
| 232 | + install_keyword("all_tg_pt", &ovr_all_tg_pt_handler, &snprint_ovr_all_tg_pt); |
| 233 | |
| 234 | install_keyword_root("multipaths", &multipaths_handler); |
| 235 | install_keyword_multi("multipath", &multipath_handler, NULL); |
| 236 | diff --git a/libmultipath/propsel.c b/libmultipath/propsel.c |
| 237 | index 627d366..9ca1355 100644 |
| 238 | --- a/libmultipath/propsel.c |
| 239 | +++ b/libmultipath/propsel.c |
| 240 | @@ -978,3 +978,18 @@ out: |
| 241 | pp->dev, pp->find_multipaths_timeout, origin); |
| 242 | return 0; |
| 243 | } |
| 244 | + |
| 245 | +int select_all_tg_pt (struct config *conf, struct multipath * mp) |
| 246 | +{ |
| 247 | + const char *origin; |
| 248 | + |
| 249 | + mp_set_ovr(all_tg_pt); |
| 250 | + mp_set_hwe(all_tg_pt); |
| 251 | + mp_set_conf(all_tg_pt); |
| 252 | + mp_set_default(all_tg_pt, DEFAULT_ALL_TG_PT); |
| 253 | +out: |
| 254 | + condlog(3, "%s: all_tg_pt = %s %s", mp->alias, |
| 255 | + (mp->all_tg_pt == ALL_TG_PT_ON)? "yes" : "no", |
| 256 | + origin); |
| 257 | + return 0; |
| 258 | +} |
| 259 | diff --git a/libmultipath/propsel.h b/libmultipath/propsel.h |
| 260 | index a022bee..ae99b92 100644 |
| 261 | --- a/libmultipath/propsel.h |
| 262 | +++ b/libmultipath/propsel.h |
| 263 | @@ -34,3 +34,4 @@ int select_ghost_delay(struct config *conf, struct multipath * mp); |
| 264 | void reconcile_features_with_options(const char *id, char **features, |
| 265 | int* no_path_retry, |
| 266 | int *retain_hwhandler); |
| 267 | +int select_all_tg_pt (struct config *conf, struct multipath * mp); |
| 268 | diff --git a/libmultipath/structs.h b/libmultipath/structs.h |
| 269 | index e424b15..0194b1e 100644 |
| 270 | --- a/libmultipath/structs.h |
| 271 | +++ b/libmultipath/structs.h |
| 272 | @@ -217,6 +217,12 @@ enum prkey_sources { |
| 273 | PRKEY_SOURCE_FILE, |
| 274 | }; |
| 275 | |
| 276 | +enum all_tg_pt_states { |
| 277 | + ALL_TG_PT_UNDEF = YNU_UNDEF, |
| 278 | + ALL_TG_PT_OFF = YNU_NO, |
| 279 | + ALL_TG_PT_ON = YNU_YES, |
| 280 | +}; |
| 281 | + |
| 282 | struct sg_id { |
| 283 | int host_no; |
| 284 | int channel; |
| 285 | @@ -362,6 +368,7 @@ struct multipath { |
| 286 | int prkey_source; |
| 287 | struct be64 reservation_key; |
| 288 | unsigned char prflag; |
| 289 | + int all_tg_pt; |
| 290 | struct gen_multipath generic_mp; |
| 291 | }; |
| 292 | |
| 293 | diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5 |
| 294 | index 96d1b66..0c1f174 100644 |
| 295 | --- a/multipath/multipath.conf.5 |
| 296 | +++ b/multipath/multipath.conf.5 |
| 297 | @@ -743,6 +743,17 @@ The default is: \fB<unset>\fR |
| 298 | . |
| 299 | . |
| 300 | .TP |
| 301 | +.B all_tg_pt |
| 302 | +This must be set to \fByes\fR to successfully use mpathpersist on arrays that |
| 303 | +automatically set and clear registration keys on all target ports from a |
| 304 | +host, instead of per target port per host. |
| 305 | +.RS |
| 306 | +.TP |
| 307 | +The default is: \fBno\fR |
| 308 | +.RE |
| 309 | +. |
| 310 | +. |
| 311 | +.TP |
| 312 | .B retain_attached_hw_handler |
| 313 | (Obsolete for kernels >= 4.3) If set to |
| 314 | .I yes |
| 315 | -- |
| 316 | 2.7.4 |
| 317 | |