From mboxrd@z Thu Jan 1 00:00:00 1970 Content-Type: multipart/mixed; boundary="===============5256022006802744135==" MIME-Version: 1.0 From: Dan Carpenter Subject: [linux-next:master 4108/4964] kernel/bpf/hashtab.c:1341 __htab_map_lookup_and_delete_batch() warn: potential spectre issue 'htab->buckets' [r] (local cap) Date: Mon, 02 Mar 2020 11:00:59 +0300 Message-ID: <20200302080059.GC4140@kadam> List-Id: To: kbuild@lists.01.org --===============5256022006802744135== Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable tree: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git= master head: 10569a280f259f696c0b32fc1d45866d2fd33f53 commit: 085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 [4108/4964] bpf: Use recur= sion prevention helpers in hashtab code :::::: branch date: 16 hours ago :::::: commit date: 3 days ago If you fix the issue, kindly add following tag Reported-by: kbuild test robot Reported-by: Dan Carpenter smatch warnings: kernel/bpf/hashtab.c:1341 __htab_map_lookup_and_delete_batch() warn: potent= ial spectre issue 'htab->buckets' [r] (local cap) # https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/commi= t/?id=3D085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 git remote add linux-next https://git.kernel.org/pub/scm/linux/kernel/git/n= ext/linux-next.git git remote update linux-next git checkout 085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 vim +1341 kernel/bpf/hashtab.c 057996380a42bb Yonghong Song 2020-01-15 1265 static int 057996380a42bb Yonghong Song 2020-01-15 1266 __htab_map_lookup_and_dele= te_batch(struct bpf_map *map, 057996380a42bb Yonghong Song 2020-01-15 1267 const union bpf_att= r *attr, 057996380a42bb Yonghong Song 2020-01-15 1268 union bpf_attr __us= er *uattr, 057996380a42bb Yonghong Song 2020-01-15 1269 bool do_delete, boo= l is_lru_map, 057996380a42bb Yonghong Song 2020-01-15 1270 bool is_percpu) 057996380a42bb Yonghong Song 2020-01-15 1271 { 057996380a42bb Yonghong Song 2020-01-15 1272 struct bpf_htab *htab =3D= container_of(map, struct bpf_htab, map); 057996380a42bb Yonghong Song 2020-01-15 1273 u32 bucket_cnt, total, ke= y_size, value_size, roundup_key_size; 057996380a42bb Yonghong Song 2020-01-15 1274 void *keys =3D NULL, *val= ues =3D NULL, *value, *dst_key, *dst_val; 057996380a42bb Yonghong Song 2020-01-15 1275 void __user *uvalues =3D = u64_to_user_ptr(attr->batch.values); 057996380a42bb Yonghong Song 2020-01-15 1276 void __user *ukeys =3D u6= 4_to_user_ptr(attr->batch.keys); 057996380a42bb Yonghong Song 2020-01-15 1277 void *ubatch =3D u64_to_u= ser_ptr(attr->batch.in_batch); 057996380a42bb Yonghong Song 2020-01-15 1278 u32 batch, max_count, siz= e, bucket_size; b9aff38de2cb16 Yonghong Song 2020-02-19 1279 struct htab_elem *node_to= _free =3D NULL; 057996380a42bb Yonghong Song 2020-01-15 1280 u64 elem_map_flags, map_f= lags; 057996380a42bb Yonghong Song 2020-01-15 1281 struct hlist_nulls_head *= head; 057996380a42bb Yonghong Song 2020-01-15 1282 struct hlist_nulls_node *= n; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1283 unsigned long flags =3D 0; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1284 bool locked =3D false; 057996380a42bb Yonghong Song 2020-01-15 1285 struct htab_elem *l; 057996380a42bb Yonghong Song 2020-01-15 1286 struct bucket *b; 057996380a42bb Yonghong Song 2020-01-15 1287 int ret =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1288 = 057996380a42bb Yonghong Song 2020-01-15 1289 elem_map_flags =3D attr->= batch.elem_flags; 057996380a42bb Yonghong Song 2020-01-15 1290 if ((elem_map_flags & ~BP= F_F_LOCK) || 057996380a42bb Yonghong Song 2020-01-15 1291 ((elem_map_flags & BP= F_F_LOCK) && !map_value_has_spin_lock(map))) 057996380a42bb Yonghong Song 2020-01-15 1292 return -EINVAL; 057996380a42bb Yonghong Song 2020-01-15 1293 = 057996380a42bb Yonghong Song 2020-01-15 1294 map_flags =3D attr->batch= .flags; 057996380a42bb Yonghong Song 2020-01-15 1295 if (map_flags) 057996380a42bb Yonghong Song 2020-01-15 1296 return -EINVAL; 057996380a42bb Yonghong Song 2020-01-15 1297 = 057996380a42bb Yonghong Song 2020-01-15 1298 max_count =3D attr->batch= .count; 057996380a42bb Yonghong Song 2020-01-15 1299 if (!max_count) 057996380a42bb Yonghong Song 2020-01-15 1300 return 0; 057996380a42bb Yonghong Song 2020-01-15 1301 = 057996380a42bb Yonghong Song 2020-01-15 1302 if (put_user(0, &uattr->b= atch.count)) 057996380a42bb Yonghong Song 2020-01-15 1303 return -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1304 = 057996380a42bb Yonghong Song 2020-01-15 1305 batch =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1306 if (ubatch && copy_from_u= ser(&batch, ubatch, sizeof(batch))) 057996380a42bb Yonghong Song 2020-01-15 1307 return -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1308 = 057996380a42bb Yonghong Song 2020-01-15 1309 if (batch >=3D htab->n_bu= ckets) This this use array_index_nospec() some how? 057996380a42bb Yonghong Song 2020-01-15 1310 return -ENOENT; 057996380a42bb Yonghong Song 2020-01-15 1311 = 057996380a42bb Yonghong Song 2020-01-15 1312 key_size =3D htab->map.ke= y_size; 057996380a42bb Yonghong Song 2020-01-15 1313 roundup_key_size =3D roun= d_up(htab->map.key_size, 8); 057996380a42bb Yonghong Song 2020-01-15 1314 value_size =3D htab->map.= value_size; 057996380a42bb Yonghong Song 2020-01-15 1315 size =3D round_up(value_s= ize, 8); 057996380a42bb Yonghong Song 2020-01-15 1316 if (is_percpu) 057996380a42bb Yonghong Song 2020-01-15 1317 value_size =3D size * nu= m_possible_cpus(); 057996380a42bb Yonghong Song 2020-01-15 1318 total =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1319 /* while experimenting wi= th hash tables with sizes ranging from 10 to 057996380a42bb Yonghong Song 2020-01-15 1320 * 1000, it was observed = that a bucket can have upto 5 entries. 057996380a42bb Yonghong Song 2020-01-15 1321 */ 057996380a42bb Yonghong Song 2020-01-15 1322 bucket_size =3D 5; 057996380a42bb Yonghong Song 2020-01-15 1323 = 057996380a42bb Yonghong Song 2020-01-15 1324 alloc: 057996380a42bb Yonghong Song 2020-01-15 1325 /* We cannot do copy_from= _user or copy_to_user inside 057996380a42bb Yonghong Song 2020-01-15 1326 * the rcu_read_lock. All= ocate enough space here. 057996380a42bb Yonghong Song 2020-01-15 1327 */ 057996380a42bb Yonghong Song 2020-01-15 1328 keys =3D kvmalloc(key_siz= e * bucket_size, GFP_USER | __GFP_NOWARN); 057996380a42bb Yonghong Song 2020-01-15 1329 values =3D kvmalloc(value= _size * bucket_size, GFP_USER | __GFP_NOWARN); 057996380a42bb Yonghong Song 2020-01-15 1330 if (!keys || !values) { 057996380a42bb Yonghong Song 2020-01-15 1331 ret =3D -ENOMEM; 057996380a42bb Yonghong Song 2020-01-15 1332 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1333 } 057996380a42bb Yonghong Song 2020-01-15 1334 = 057996380a42bb Yonghong Song 2020-01-15 1335 again: 085fee1a72a9fb Thomas Gleixner 2020-02-24 1336 bpf_disable_instrumentati= on(); 057996380a42bb Yonghong Song 2020-01-15 1337 rcu_read_lock(); 057996380a42bb Yonghong Song 2020-01-15 1338 again_nocopy: 057996380a42bb Yonghong Song 2020-01-15 1339 dst_key =3D keys; 057996380a42bb Yonghong Song 2020-01-15 1340 dst_val =3D values; 057996380a42bb Yonghong Song 2020-01-15 @1341 b =3D &htab->buckets[batc= h]; 057996380a42bb Yonghong Song 2020-01-15 1342 head =3D &b->head; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1343 /* do not grab the lock u= nless need it (bucket_cnt > 0). */ 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1344 if (locked) 057996380a42bb Yonghong Song 2020-01-15 1345 raw_spin_lock_irqsave(&b= ->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1346 = 057996380a42bb Yonghong Song 2020-01-15 1347 bucket_cnt =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1348 hlist_nulls_for_each_entr= y_rcu(l, n, head, hash_node) 057996380a42bb Yonghong Song 2020-01-15 1349 bucket_cnt++; 057996380a42bb Yonghong Song 2020-01-15 1350 = 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1351 if (bucket_cnt && !locked= ) { 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1352 locked =3D true; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1353 goto again_nocopy; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1354 } 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1355 = 057996380a42bb Yonghong Song 2020-01-15 1356 if (bucket_cnt > (max_cou= nt - total)) { 057996380a42bb Yonghong Song 2020-01-15 1357 if (total =3D=3D 0) 057996380a42bb Yonghong Song 2020-01-15 1358 ret =3D -ENOSPC; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1359 /* Note that since bucke= t_cnt > 0 here, it is implicit 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1360 * that the locked was g= rabbed, so release it. 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1361 */ 057996380a42bb Yonghong Song 2020-01-15 1362 raw_spin_unlock_irqresto= re(&b->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1363 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1364 bpf_enable_instrumentati= on(); 057996380a42bb Yonghong Song 2020-01-15 1365 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1366 } 057996380a42bb Yonghong Song 2020-01-15 1367 = 057996380a42bb Yonghong Song 2020-01-15 1368 if (bucket_cnt > bucket_s= ize) { 057996380a42bb Yonghong Song 2020-01-15 1369 bucket_size =3D bucket_c= nt; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1370 /* Note that since bucke= t_cnt > 0 here, it is implicit 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1371 * that the locked was g= rabbed, so release it. 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1372 */ 057996380a42bb Yonghong Song 2020-01-15 1373 raw_spin_unlock_irqresto= re(&b->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1374 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1375 bpf_enable_instrumentati= on(); 057996380a42bb Yonghong Song 2020-01-15 1376 kvfree(keys); 057996380a42bb Yonghong Song 2020-01-15 1377 kvfree(values); 057996380a42bb Yonghong Song 2020-01-15 1378 goto alloc; 057996380a42bb Yonghong Song 2020-01-15 1379 } 057996380a42bb Yonghong Song 2020-01-15 1380 = 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1381 /* Next block is only saf= e to run if you have grabbed the lock */ 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1382 if (!locked) 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1383 goto next_batch; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1384 = 057996380a42bb Yonghong Song 2020-01-15 1385 hlist_nulls_for_each_entr= y_safe(l, n, head, hash_node) { 057996380a42bb Yonghong Song 2020-01-15 1386 memcpy(dst_key, l->key, = key_size); 057996380a42bb Yonghong Song 2020-01-15 1387 = 057996380a42bb Yonghong Song 2020-01-15 1388 if (is_percpu) { 057996380a42bb Yonghong Song 2020-01-15 1389 int off =3D 0, cpu; 057996380a42bb Yonghong Song 2020-01-15 1390 void __percpu *pptr; 057996380a42bb Yonghong Song 2020-01-15 1391 = 057996380a42bb Yonghong Song 2020-01-15 1392 pptr =3D htab_elem_get_= ptr(l, map->key_size); 057996380a42bb Yonghong Song 2020-01-15 1393 for_each_possible_cpu(c= pu) { 057996380a42bb Yonghong Song 2020-01-15 1394 bpf_long_memcpy(dst_va= l + off, 057996380a42bb Yonghong Song 2020-01-15 1395 per_cpu_ptr(pptr, cp= u), size); 057996380a42bb Yonghong Song 2020-01-15 1396 off +=3D size; 057996380a42bb Yonghong Song 2020-01-15 1397 } 057996380a42bb Yonghong Song 2020-01-15 1398 } else { 057996380a42bb Yonghong Song 2020-01-15 1399 value =3D l->key + roun= dup_key_size; 057996380a42bb Yonghong Song 2020-01-15 1400 if (elem_map_flags & BP= F_F_LOCK) 057996380a42bb Yonghong Song 2020-01-15 1401 copy_map_value_locked(= map, dst_val, value, 057996380a42bb Yonghong Song 2020-01-15 1402 true); 057996380a42bb Yonghong Song 2020-01-15 1403 else 057996380a42bb Yonghong Song 2020-01-15 1404 copy_map_value(map, ds= t_val, value); 057996380a42bb Yonghong Song 2020-01-15 1405 check_and_init_map_lock= (map, dst_val); 057996380a42bb Yonghong Song 2020-01-15 1406 } 057996380a42bb Yonghong Song 2020-01-15 1407 if (do_delete) { 057996380a42bb Yonghong Song 2020-01-15 1408 hlist_nulls_del_rcu(&l-= >hash_node); b9aff38de2cb16 Yonghong Song 2020-02-19 1409 = b9aff38de2cb16 Yonghong Song 2020-02-19 1410 /* bpf_lru_push_free() = will acquire lru_lock, which b9aff38de2cb16 Yonghong Song 2020-02-19 1411 * may cause deadlock. = See comments in function b9aff38de2cb16 Yonghong Song 2020-02-19 1412 * prealloc_lru_pop(). = Let us do bpf_lru_push_free() b9aff38de2cb16 Yonghong Song 2020-02-19 1413 * after releasing the = bucket lock. b9aff38de2cb16 Yonghong Song 2020-02-19 1414 */ b9aff38de2cb16 Yonghong Song 2020-02-19 1415 if (is_lru_map) { b9aff38de2cb16 Yonghong Song 2020-02-19 1416 l->batch_flink =3D nod= e_to_free; b9aff38de2cb16 Yonghong Song 2020-02-19 1417 node_to_free =3D l; b9aff38de2cb16 Yonghong Song 2020-02-19 1418 } else { 057996380a42bb Yonghong Song 2020-01-15 1419 free_htab_elem(htab, l= ); 057996380a42bb Yonghong Song 2020-01-15 1420 } b9aff38de2cb16 Yonghong Song 2020-02-19 1421 } 057996380a42bb Yonghong Song 2020-01-15 1422 dst_key +=3D key_size; 057996380a42bb Yonghong Song 2020-01-15 1423 dst_val +=3D value_size; 057996380a42bb Yonghong Song 2020-01-15 1424 } 057996380a42bb Yonghong Song 2020-01-15 1425 = 057996380a42bb Yonghong Song 2020-01-15 1426 raw_spin_unlock_irqrestor= e(&b->lock, flags); 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1427 locked =3D false; b9aff38de2cb16 Yonghong Song 2020-02-19 1428 = b9aff38de2cb16 Yonghong Song 2020-02-19 1429 while (node_to_free) { b9aff38de2cb16 Yonghong Song 2020-02-19 1430 l =3D node_to_free; b9aff38de2cb16 Yonghong Song 2020-02-19 1431 node_to_free =3D node_to= _free->batch_flink; b9aff38de2cb16 Yonghong Song 2020-02-19 1432 bpf_lru_push_free(&htab-= >lru, &l->lru_node); b9aff38de2cb16 Yonghong Song 2020-02-19 1433 } b9aff38de2cb16 Yonghong Song 2020-02-19 1434 = 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1435 next_batch: 057996380a42bb Yonghong Song 2020-01-15 1436 /* If we are not copying = data, we can go to next bucket and avoid 057996380a42bb Yonghong Song 2020-01-15 1437 * unlocking the rcu. 057996380a42bb Yonghong Song 2020-01-15 1438 */ 057996380a42bb Yonghong Song 2020-01-15 1439 if (!bucket_cnt && (batch= + 1 < htab->n_buckets)) { 057996380a42bb Yonghong Song 2020-01-15 1440 batch++; 057996380a42bb Yonghong Song 2020-01-15 1441 goto again_nocopy; 057996380a42bb Yonghong Song 2020-01-15 1442 } 057996380a42bb Yonghong Song 2020-01-15 1443 = 057996380a42bb Yonghong Song 2020-01-15 1444 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1445 bpf_enable_instrumentatio= n(); 057996380a42bb Yonghong Song 2020-01-15 1446 if (bucket_cnt && (copy_t= o_user(ukeys + total * key_size, keys, 057996380a42bb Yonghong Song 2020-01-15 1447 key_size * bucket_cnt= ) || 057996380a42bb Yonghong Song 2020-01-15 1448 copy_to_user(uvalues = + total * value_size, values, 057996380a42bb Yonghong Song 2020-01-15 1449 value_size * bucket_c= nt))) { 057996380a42bb Yonghong Song 2020-01-15 1450 ret =3D -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1451 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1452 } 057996380a42bb Yonghong Song 2020-01-15 1453 = 057996380a42bb Yonghong Song 2020-01-15 1454 total +=3D bucket_cnt; 057996380a42bb Yonghong Song 2020-01-15 1455 batch++; 057996380a42bb Yonghong Song 2020-01-15 1456 if (batch >=3D htab->n_bu= ckets) { 057996380a42bb Yonghong Song 2020-01-15 1457 ret =3D -ENOENT; 057996380a42bb Yonghong Song 2020-01-15 1458 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1459 } 057996380a42bb Yonghong Song 2020-01-15 1460 goto again; 057996380a42bb Yonghong Song 2020-01-15 1461 = 057996380a42bb Yonghong Song 2020-01-15 1462 after_loop: 057996380a42bb Yonghong Song 2020-01-15 1463 if (ret =3D=3D -EFAULT) 057996380a42bb Yonghong Song 2020-01-15 1464 goto out; 057996380a42bb Yonghong Song 2020-01-15 1465 = 057996380a42bb Yonghong Song 2020-01-15 1466 /* copy # of entries and = next batch */ 057996380a42bb Yonghong Song 2020-01-15 1467 ubatch =3D u64_to_user_pt= r(attr->batch.out_batch); 057996380a42bb Yonghong Song 2020-01-15 1468 if (copy_to_user(ubatch, = &batch, sizeof(batch)) || 057996380a42bb Yonghong Song 2020-01-15 1469 put_user(total, &uatt= r->batch.count)) 057996380a42bb Yonghong Song 2020-01-15 1470 ret =3D -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1471 = 057996380a42bb Yonghong Song 2020-01-15 1472 out: 057996380a42bb Yonghong Song 2020-01-15 1473 kvfree(keys); 057996380a42bb Yonghong Song 2020-01-15 1474 kvfree(values); 057996380a42bb Yonghong Song 2020-01-15 1475 return ret; 057996380a42bb Yonghong Song 2020-01-15 1476 } --- 0-DAY CI Kernel Test Service, Intel Corporation https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org --===============5256022006802744135==-- From mboxrd@z Thu Jan 1 00:00:00 1970 Content-Type: multipart/mixed; boundary="===============8550214704122596140==" MIME-Version: 1.0 From: Dan Carpenter To: kbuild-all@lists.01.org Subject: [linux-next:master 4108/4964] kernel/bpf/hashtab.c:1341 __htab_map_lookup_and_delete_batch() warn: potential spectre issue 'htab->buckets' [r] (local cap) Date: Mon, 02 Mar 2020 11:00:59 +0300 Message-ID: <20200302080059.GC4140@kadam> List-Id: --===============8550214704122596140== Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable tree: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git= master head: 10569a280f259f696c0b32fc1d45866d2fd33f53 commit: 085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 [4108/4964] bpf: Use recur= sion prevention helpers in hashtab code :::::: branch date: 16 hours ago :::::: commit date: 3 days ago If you fix the issue, kindly add following tag Reported-by: kbuild test robot Reported-by: Dan Carpenter smatch warnings: kernel/bpf/hashtab.c:1341 __htab_map_lookup_and_delete_batch() warn: potent= ial spectre issue 'htab->buckets' [r] (local cap) # https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/commi= t/?id=3D085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 git remote add linux-next https://git.kernel.org/pub/scm/linux/kernel/git/n= ext/linux-next.git git remote update linux-next git checkout 085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 vim +1341 kernel/bpf/hashtab.c 057996380a42bb Yonghong Song 2020-01-15 1265 static int 057996380a42bb Yonghong Song 2020-01-15 1266 __htab_map_lookup_and_dele= te_batch(struct bpf_map *map, 057996380a42bb Yonghong Song 2020-01-15 1267 const union bpf_att= r *attr, 057996380a42bb Yonghong Song 2020-01-15 1268 union bpf_attr __us= er *uattr, 057996380a42bb Yonghong Song 2020-01-15 1269 bool do_delete, boo= l is_lru_map, 057996380a42bb Yonghong Song 2020-01-15 1270 bool is_percpu) 057996380a42bb Yonghong Song 2020-01-15 1271 { 057996380a42bb Yonghong Song 2020-01-15 1272 struct bpf_htab *htab =3D= container_of(map, struct bpf_htab, map); 057996380a42bb Yonghong Song 2020-01-15 1273 u32 bucket_cnt, total, ke= y_size, value_size, roundup_key_size; 057996380a42bb Yonghong Song 2020-01-15 1274 void *keys =3D NULL, *val= ues =3D NULL, *value, *dst_key, *dst_val; 057996380a42bb Yonghong Song 2020-01-15 1275 void __user *uvalues =3D = u64_to_user_ptr(attr->batch.values); 057996380a42bb Yonghong Song 2020-01-15 1276 void __user *ukeys =3D u6= 4_to_user_ptr(attr->batch.keys); 057996380a42bb Yonghong Song 2020-01-15 1277 void *ubatch =3D u64_to_u= ser_ptr(attr->batch.in_batch); 057996380a42bb Yonghong Song 2020-01-15 1278 u32 batch, max_count, siz= e, bucket_size; b9aff38de2cb16 Yonghong Song 2020-02-19 1279 struct htab_elem *node_to= _free =3D NULL; 057996380a42bb Yonghong Song 2020-01-15 1280 u64 elem_map_flags, map_f= lags; 057996380a42bb Yonghong Song 2020-01-15 1281 struct hlist_nulls_head *= head; 057996380a42bb Yonghong Song 2020-01-15 1282 struct hlist_nulls_node *= n; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1283 unsigned long flags =3D 0; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1284 bool locked =3D false; 057996380a42bb Yonghong Song 2020-01-15 1285 struct htab_elem *l; 057996380a42bb Yonghong Song 2020-01-15 1286 struct bucket *b; 057996380a42bb Yonghong Song 2020-01-15 1287 int ret =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1288 = 057996380a42bb Yonghong Song 2020-01-15 1289 elem_map_flags =3D attr->= batch.elem_flags; 057996380a42bb Yonghong Song 2020-01-15 1290 if ((elem_map_flags & ~BP= F_F_LOCK) || 057996380a42bb Yonghong Song 2020-01-15 1291 ((elem_map_flags & BP= F_F_LOCK) && !map_value_has_spin_lock(map))) 057996380a42bb Yonghong Song 2020-01-15 1292 return -EINVAL; 057996380a42bb Yonghong Song 2020-01-15 1293 = 057996380a42bb Yonghong Song 2020-01-15 1294 map_flags =3D attr->batch= .flags; 057996380a42bb Yonghong Song 2020-01-15 1295 if (map_flags) 057996380a42bb Yonghong Song 2020-01-15 1296 return -EINVAL; 057996380a42bb Yonghong Song 2020-01-15 1297 = 057996380a42bb Yonghong Song 2020-01-15 1298 max_count =3D attr->batch= .count; 057996380a42bb Yonghong Song 2020-01-15 1299 if (!max_count) 057996380a42bb Yonghong Song 2020-01-15 1300 return 0; 057996380a42bb Yonghong Song 2020-01-15 1301 = 057996380a42bb Yonghong Song 2020-01-15 1302 if (put_user(0, &uattr->b= atch.count)) 057996380a42bb Yonghong Song 2020-01-15 1303 return -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1304 = 057996380a42bb Yonghong Song 2020-01-15 1305 batch =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1306 if (ubatch && copy_from_u= ser(&batch, ubatch, sizeof(batch))) 057996380a42bb Yonghong Song 2020-01-15 1307 return -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1308 = 057996380a42bb Yonghong Song 2020-01-15 1309 if (batch >=3D htab->n_bu= ckets) This this use array_index_nospec() some how? 057996380a42bb Yonghong Song 2020-01-15 1310 return -ENOENT; 057996380a42bb Yonghong Song 2020-01-15 1311 = 057996380a42bb Yonghong Song 2020-01-15 1312 key_size =3D htab->map.ke= y_size; 057996380a42bb Yonghong Song 2020-01-15 1313 roundup_key_size =3D roun= d_up(htab->map.key_size, 8); 057996380a42bb Yonghong Song 2020-01-15 1314 value_size =3D htab->map.= value_size; 057996380a42bb Yonghong Song 2020-01-15 1315 size =3D round_up(value_s= ize, 8); 057996380a42bb Yonghong Song 2020-01-15 1316 if (is_percpu) 057996380a42bb Yonghong Song 2020-01-15 1317 value_size =3D size * nu= m_possible_cpus(); 057996380a42bb Yonghong Song 2020-01-15 1318 total =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1319 /* while experimenting wi= th hash tables with sizes ranging from 10 to 057996380a42bb Yonghong Song 2020-01-15 1320 * 1000, it was observed = that a bucket can have upto 5 entries. 057996380a42bb Yonghong Song 2020-01-15 1321 */ 057996380a42bb Yonghong Song 2020-01-15 1322 bucket_size =3D 5; 057996380a42bb Yonghong Song 2020-01-15 1323 = 057996380a42bb Yonghong Song 2020-01-15 1324 alloc: 057996380a42bb Yonghong Song 2020-01-15 1325 /* We cannot do copy_from= _user or copy_to_user inside 057996380a42bb Yonghong Song 2020-01-15 1326 * the rcu_read_lock. All= ocate enough space here. 057996380a42bb Yonghong Song 2020-01-15 1327 */ 057996380a42bb Yonghong Song 2020-01-15 1328 keys =3D kvmalloc(key_siz= e * bucket_size, GFP_USER | __GFP_NOWARN); 057996380a42bb Yonghong Song 2020-01-15 1329 values =3D kvmalloc(value= _size * bucket_size, GFP_USER | __GFP_NOWARN); 057996380a42bb Yonghong Song 2020-01-15 1330 if (!keys || !values) { 057996380a42bb Yonghong Song 2020-01-15 1331 ret =3D -ENOMEM; 057996380a42bb Yonghong Song 2020-01-15 1332 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1333 } 057996380a42bb Yonghong Song 2020-01-15 1334 = 057996380a42bb Yonghong Song 2020-01-15 1335 again: 085fee1a72a9fb Thomas Gleixner 2020-02-24 1336 bpf_disable_instrumentati= on(); 057996380a42bb Yonghong Song 2020-01-15 1337 rcu_read_lock(); 057996380a42bb Yonghong Song 2020-01-15 1338 again_nocopy: 057996380a42bb Yonghong Song 2020-01-15 1339 dst_key =3D keys; 057996380a42bb Yonghong Song 2020-01-15 1340 dst_val =3D values; 057996380a42bb Yonghong Song 2020-01-15 @1341 b =3D &htab->buckets[batc= h]; 057996380a42bb Yonghong Song 2020-01-15 1342 head =3D &b->head; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1343 /* do not grab the lock u= nless need it (bucket_cnt > 0). */ 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1344 if (locked) 057996380a42bb Yonghong Song 2020-01-15 1345 raw_spin_lock_irqsave(&b= ->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1346 = 057996380a42bb Yonghong Song 2020-01-15 1347 bucket_cnt =3D 0; 057996380a42bb Yonghong Song 2020-01-15 1348 hlist_nulls_for_each_entr= y_rcu(l, n, head, hash_node) 057996380a42bb Yonghong Song 2020-01-15 1349 bucket_cnt++; 057996380a42bb Yonghong Song 2020-01-15 1350 = 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1351 if (bucket_cnt && !locked= ) { 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1352 locked =3D true; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1353 goto again_nocopy; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1354 } 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1355 = 057996380a42bb Yonghong Song 2020-01-15 1356 if (bucket_cnt > (max_cou= nt - total)) { 057996380a42bb Yonghong Song 2020-01-15 1357 if (total =3D=3D 0) 057996380a42bb Yonghong Song 2020-01-15 1358 ret =3D -ENOSPC; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1359 /* Note that since bucke= t_cnt > 0 here, it is implicit 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1360 * that the locked was g= rabbed, so release it. 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1361 */ 057996380a42bb Yonghong Song 2020-01-15 1362 raw_spin_unlock_irqresto= re(&b->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1363 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1364 bpf_enable_instrumentati= on(); 057996380a42bb Yonghong Song 2020-01-15 1365 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1366 } 057996380a42bb Yonghong Song 2020-01-15 1367 = 057996380a42bb Yonghong Song 2020-01-15 1368 if (bucket_cnt > bucket_s= ize) { 057996380a42bb Yonghong Song 2020-01-15 1369 bucket_size =3D bucket_c= nt; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1370 /* Note that since bucke= t_cnt > 0 here, it is implicit 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1371 * that the locked was g= rabbed, so release it. 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1372 */ 057996380a42bb Yonghong Song 2020-01-15 1373 raw_spin_unlock_irqresto= re(&b->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1374 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1375 bpf_enable_instrumentati= on(); 057996380a42bb Yonghong Song 2020-01-15 1376 kvfree(keys); 057996380a42bb Yonghong Song 2020-01-15 1377 kvfree(values); 057996380a42bb Yonghong Song 2020-01-15 1378 goto alloc; 057996380a42bb Yonghong Song 2020-01-15 1379 } 057996380a42bb Yonghong Song 2020-01-15 1380 = 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1381 /* Next block is only saf= e to run if you have grabbed the lock */ 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1382 if (!locked) 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1383 goto next_batch; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1384 = 057996380a42bb Yonghong Song 2020-01-15 1385 hlist_nulls_for_each_entr= y_safe(l, n, head, hash_node) { 057996380a42bb Yonghong Song 2020-01-15 1386 memcpy(dst_key, l->key, = key_size); 057996380a42bb Yonghong Song 2020-01-15 1387 = 057996380a42bb Yonghong Song 2020-01-15 1388 if (is_percpu) { 057996380a42bb Yonghong Song 2020-01-15 1389 int off =3D 0, cpu; 057996380a42bb Yonghong Song 2020-01-15 1390 void __percpu *pptr; 057996380a42bb Yonghong Song 2020-01-15 1391 = 057996380a42bb Yonghong Song 2020-01-15 1392 pptr =3D htab_elem_get_= ptr(l, map->key_size); 057996380a42bb Yonghong Song 2020-01-15 1393 for_each_possible_cpu(c= pu) { 057996380a42bb Yonghong Song 2020-01-15 1394 bpf_long_memcpy(dst_va= l + off, 057996380a42bb Yonghong Song 2020-01-15 1395 per_cpu_ptr(pptr, cp= u), size); 057996380a42bb Yonghong Song 2020-01-15 1396 off +=3D size; 057996380a42bb Yonghong Song 2020-01-15 1397 } 057996380a42bb Yonghong Song 2020-01-15 1398 } else { 057996380a42bb Yonghong Song 2020-01-15 1399 value =3D l->key + roun= dup_key_size; 057996380a42bb Yonghong Song 2020-01-15 1400 if (elem_map_flags & BP= F_F_LOCK) 057996380a42bb Yonghong Song 2020-01-15 1401 copy_map_value_locked(= map, dst_val, value, 057996380a42bb Yonghong Song 2020-01-15 1402 true); 057996380a42bb Yonghong Song 2020-01-15 1403 else 057996380a42bb Yonghong Song 2020-01-15 1404 copy_map_value(map, ds= t_val, value); 057996380a42bb Yonghong Song 2020-01-15 1405 check_and_init_map_lock= (map, dst_val); 057996380a42bb Yonghong Song 2020-01-15 1406 } 057996380a42bb Yonghong Song 2020-01-15 1407 if (do_delete) { 057996380a42bb Yonghong Song 2020-01-15 1408 hlist_nulls_del_rcu(&l-= >hash_node); b9aff38de2cb16 Yonghong Song 2020-02-19 1409 = b9aff38de2cb16 Yonghong Song 2020-02-19 1410 /* bpf_lru_push_free() = will acquire lru_lock, which b9aff38de2cb16 Yonghong Song 2020-02-19 1411 * may cause deadlock. = See comments in function b9aff38de2cb16 Yonghong Song 2020-02-19 1412 * prealloc_lru_pop(). = Let us do bpf_lru_push_free() b9aff38de2cb16 Yonghong Song 2020-02-19 1413 * after releasing the = bucket lock. b9aff38de2cb16 Yonghong Song 2020-02-19 1414 */ b9aff38de2cb16 Yonghong Song 2020-02-19 1415 if (is_lru_map) { b9aff38de2cb16 Yonghong Song 2020-02-19 1416 l->batch_flink =3D nod= e_to_free; b9aff38de2cb16 Yonghong Song 2020-02-19 1417 node_to_free =3D l; b9aff38de2cb16 Yonghong Song 2020-02-19 1418 } else { 057996380a42bb Yonghong Song 2020-01-15 1419 free_htab_elem(htab, l= ); 057996380a42bb Yonghong Song 2020-01-15 1420 } b9aff38de2cb16 Yonghong Song 2020-02-19 1421 } 057996380a42bb Yonghong Song 2020-01-15 1422 dst_key +=3D key_size; 057996380a42bb Yonghong Song 2020-01-15 1423 dst_val +=3D value_size; 057996380a42bb Yonghong Song 2020-01-15 1424 } 057996380a42bb Yonghong Song 2020-01-15 1425 = 057996380a42bb Yonghong Song 2020-01-15 1426 raw_spin_unlock_irqrestor= e(&b->lock, flags); 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1427 locked =3D false; b9aff38de2cb16 Yonghong Song 2020-02-19 1428 = b9aff38de2cb16 Yonghong Song 2020-02-19 1429 while (node_to_free) { b9aff38de2cb16 Yonghong Song 2020-02-19 1430 l =3D node_to_free; b9aff38de2cb16 Yonghong Song 2020-02-19 1431 node_to_free =3D node_to= _free->batch_flink; b9aff38de2cb16 Yonghong Song 2020-02-19 1432 bpf_lru_push_free(&htab-= >lru, &l->lru_node); b9aff38de2cb16 Yonghong Song 2020-02-19 1433 } b9aff38de2cb16 Yonghong Song 2020-02-19 1434 = 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1435 next_batch: 057996380a42bb Yonghong Song 2020-01-15 1436 /* If we are not copying = data, we can go to next bucket and avoid 057996380a42bb Yonghong Song 2020-01-15 1437 * unlocking the rcu. 057996380a42bb Yonghong Song 2020-01-15 1438 */ 057996380a42bb Yonghong Song 2020-01-15 1439 if (!bucket_cnt && (batch= + 1 < htab->n_buckets)) { 057996380a42bb Yonghong Song 2020-01-15 1440 batch++; 057996380a42bb Yonghong Song 2020-01-15 1441 goto again_nocopy; 057996380a42bb Yonghong Song 2020-01-15 1442 } 057996380a42bb Yonghong Song 2020-01-15 1443 = 057996380a42bb Yonghong Song 2020-01-15 1444 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1445 bpf_enable_instrumentatio= n(); 057996380a42bb Yonghong Song 2020-01-15 1446 if (bucket_cnt && (copy_t= o_user(ukeys + total * key_size, keys, 057996380a42bb Yonghong Song 2020-01-15 1447 key_size * bucket_cnt= ) || 057996380a42bb Yonghong Song 2020-01-15 1448 copy_to_user(uvalues = + total * value_size, values, 057996380a42bb Yonghong Song 2020-01-15 1449 value_size * bucket_c= nt))) { 057996380a42bb Yonghong Song 2020-01-15 1450 ret =3D -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1451 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1452 } 057996380a42bb Yonghong Song 2020-01-15 1453 = 057996380a42bb Yonghong Song 2020-01-15 1454 total +=3D bucket_cnt; 057996380a42bb Yonghong Song 2020-01-15 1455 batch++; 057996380a42bb Yonghong Song 2020-01-15 1456 if (batch >=3D htab->n_bu= ckets) { 057996380a42bb Yonghong Song 2020-01-15 1457 ret =3D -ENOENT; 057996380a42bb Yonghong Song 2020-01-15 1458 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1459 } 057996380a42bb Yonghong Song 2020-01-15 1460 goto again; 057996380a42bb Yonghong Song 2020-01-15 1461 = 057996380a42bb Yonghong Song 2020-01-15 1462 after_loop: 057996380a42bb Yonghong Song 2020-01-15 1463 if (ret =3D=3D -EFAULT) 057996380a42bb Yonghong Song 2020-01-15 1464 goto out; 057996380a42bb Yonghong Song 2020-01-15 1465 = 057996380a42bb Yonghong Song 2020-01-15 1466 /* copy # of entries and = next batch */ 057996380a42bb Yonghong Song 2020-01-15 1467 ubatch =3D u64_to_user_pt= r(attr->batch.out_batch); 057996380a42bb Yonghong Song 2020-01-15 1468 if (copy_to_user(ubatch, = &batch, sizeof(batch)) || 057996380a42bb Yonghong Song 2020-01-15 1469 put_user(total, &uatt= r->batch.count)) 057996380a42bb Yonghong Song 2020-01-15 1470 ret =3D -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1471 = 057996380a42bb Yonghong Song 2020-01-15 1472 out: 057996380a42bb Yonghong Song 2020-01-15 1473 kvfree(keys); 057996380a42bb Yonghong Song 2020-01-15 1474 kvfree(values); 057996380a42bb Yonghong Song 2020-01-15 1475 return ret; 057996380a42bb Yonghong Song 2020-01-15 1476 } --- 0-DAY CI Kernel Test Service, Intel Corporation https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org --===============8550214704122596140==-- From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-7.3 required=3.0 tests=DKIMWL_WL_HIGH,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,HEADER_FROM_DIFFERENT_DOMAINS,MAILING_LIST_MULTI, MENTIONS_GIT_HOSTING,SPF_HELO_NONE,SPF_PASS,UNPARSEABLE_RELAY, USER_AGENT_SANE_1 autolearn=ham autolearn_force=no version=3.4.0 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99]) by smtp.lore.kernel.org (Postfix) with ESMTP id 50AFCC3F2D1 for ; Mon, 2 Mar 2020 08:01:14 +0000 (UTC) Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by mail.kernel.org (Postfix) with ESMTP id EA44124699 for ; Mon, 2 Mar 2020 08:01:13 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=pass (2048-bit key) header.d=oracle.com header.i=@oracle.com header.b="YOPbMsjq" DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org EA44124699 Authentication-Results: mail.kernel.org; dmarc=fail (p=none dis=none) header.from=oracle.com Authentication-Results: mail.kernel.org; spf=pass smtp.mailfrom=owner-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix) id 801026B0005; Mon, 2 Mar 2020 03:01:13 -0500 (EST) Received: by kanga.kvack.org (Postfix, from userid 40) id 7B03C6B0006; Mon, 2 Mar 2020 03:01:13 -0500 (EST) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 653EC6B0007; Mon, 2 Mar 2020 03:01:13 -0500 (EST) X-Delivered-To: linux-mm@kvack.org Received: from forelay.hostedemail.com (smtprelay0081.hostedemail.com [216.40.44.81]) by kanga.kvack.org (Postfix) with ESMTP id 41AD26B0005 for ; Mon, 2 Mar 2020 03:01:13 -0500 (EST) Received: from smtpin29.hostedemail.com (10.5.19.251.rfc1918.com [10.5.19.251]) by forelay03.hostedemail.com (Postfix) with ESMTP id 054C98248D7C for ; Mon, 2 Mar 2020 08:01:13 +0000 (UTC) X-FDA: 76549676826.29.toe23_a8ee636c634b X-HE-Tag: toe23_a8ee636c634b X-Filterd-Recvd-Size: 20351 Received: from aserp2120.oracle.com (aserp2120.oracle.com [141.146.126.78]) by imf04.hostedemail.com (Postfix) with ESMTP for ; Mon, 2 Mar 2020 08:01:12 +0000 (UTC) Received: from pps.filterd (aserp2120.oracle.com [127.0.0.1]) by aserp2120.oracle.com (8.16.0.42/8.16.0.42) with SMTP id 0227w9c1135661; Mon, 2 Mar 2020 08:01:09 GMT DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=oracle.com; h=date : from : to : cc : subject : message-id : mime-version : content-type; s=corp-2020-01-29; bh=wLiAOBzFyPhzC4eAJHaUnFXMeSYImhUPdLNo4ZidAdY=; b=YOPbMsjqGpjLg6Tfk/SrPh1jBbvpNSMPvpNUfh35zXgE5kB4HKKBqXP3wn6zHOwOq1Iw JxnfsvOeFLKOXcNuPtH2IVZeYmtAI02bZRLY0GFjMvnpfQhE5ZO0GrgIepLSzsz8lgsN shfBdH78E15McYFUxKabW8H/yK6yHRSmoHiihWvSdR70xl7WG7UDtLnblneVpqbpSulx +evahFQ4cNSmHdaEvQhgpN7WIoLkybgBP1jVmb5QHYkkWfnO5OSusL5mmifvSkGGyRz4 +juw85s/f+qOpGzovQlCYgvejmVXZFPCU3DmAm3I3JA1HBzD0sJlybO+/WNvopuEC3xS iw== Received: from aserp3030.oracle.com (aserp3030.oracle.com [141.146.126.71]) by aserp2120.oracle.com with ESMTP id 2yffwqe171-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=OK); Mon, 02 Mar 2020 08:01:09 +0000 Received: from pps.filterd (aserp3030.oracle.com [127.0.0.1]) by aserp3030.oracle.com (8.16.0.42/8.16.0.42) with SMTP id 0227v4Ch112543; Mon, 2 Mar 2020 08:01:09 GMT Received: from aserv0121.oracle.com (aserv0121.oracle.com [141.146.126.235]) by aserp3030.oracle.com with ESMTP id 2yg1gughd6-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=OK); Mon, 02 Mar 2020 08:01:09 +0000 Received: from abhmp0019.oracle.com (abhmp0019.oracle.com [141.146.116.25]) by aserv0121.oracle.com (8.14.4/8.13.8) with ESMTP id 022818P0009463; Mon, 2 Mar 2020 08:01:08 GMT Received: from kadam (/41.210.147.242) by default (Oracle Beehive Gateway v4.0) with ESMTP ; Mon, 02 Mar 2020 00:01:06 -0800 Date: Mon, 2 Mar 2020 11:00:59 +0300 From: Dan Carpenter To: kbuild@lists.01.org, Andrew Morton Cc: kbuild-all@lists.01.org, Linux Memory Management List Subject: [linux-next:master 4108/4964] kernel/bpf/hashtab.c:1341 __htab_map_lookup_and_delete_batch() warn: potential spectre issue 'htab->buckets' [r] (local cap) Message-ID: <20200302080059.GC4140@kadam> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline User-Agent: Mutt/1.9.4 (2018-02-28) X-Proofpoint-Virus-Version: vendor=nai engine=6000 definitions=9547 signatures=668685 X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 adultscore=0 phishscore=0 suspectscore=0 malwarescore=0 mlxlogscore=999 mlxscore=0 spamscore=0 bulkscore=0 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.12.0-2001150001 definitions=main-2003020060 X-Proofpoint-Virus-Version: vendor=nai engine=6000 definitions=9547 signatures=668685 X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 lowpriorityscore=0 spamscore=0 impostorscore=0 malwarescore=0 mlxlogscore=999 mlxscore=0 suspectscore=0 phishscore=0 clxscore=1015 bulkscore=0 adultscore=0 priorityscore=1501 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.12.0-2001150001 definitions=main-2003020060 X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: tree: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git master head: 10569a280f259f696c0b32fc1d45866d2fd33f53 commit: 085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 [4108/4964] bpf: Use recursion prevention helpers in hashtab code :::::: branch date: 16 hours ago :::::: commit date: 3 days ago If you fix the issue, kindly add following tag Reported-by: kbuild test robot Reported-by: Dan Carpenter smatch warnings: kernel/bpf/hashtab.c:1341 __htab_map_lookup_and_delete_batch() warn: potential spectre issue 'htab->buckets' [r] (local cap) # https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/commit/?id=085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 git remote add linux-next https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git git remote update linux-next git checkout 085fee1a72a9fba101a4a68a2c02fa8bd2b6f913 vim +1341 kernel/bpf/hashtab.c 057996380a42bb Yonghong Song 2020-01-15 1265 static int 057996380a42bb Yonghong Song 2020-01-15 1266 __htab_map_lookup_and_delete_batch(struct bpf_map *map, 057996380a42bb Yonghong Song 2020-01-15 1267 const union bpf_attr *attr, 057996380a42bb Yonghong Song 2020-01-15 1268 union bpf_attr __user *uattr, 057996380a42bb Yonghong Song 2020-01-15 1269 bool do_delete, bool is_lru_map, 057996380a42bb Yonghong Song 2020-01-15 1270 bool is_percpu) 057996380a42bb Yonghong Song 2020-01-15 1271 { 057996380a42bb Yonghong Song 2020-01-15 1272 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); 057996380a42bb Yonghong Song 2020-01-15 1273 u32 bucket_cnt, total, key_size, value_size, roundup_key_size; 057996380a42bb Yonghong Song 2020-01-15 1274 void *keys = NULL, *values = NULL, *value, *dst_key, *dst_val; 057996380a42bb Yonghong Song 2020-01-15 1275 void __user *uvalues = u64_to_user_ptr(attr->batch.values); 057996380a42bb Yonghong Song 2020-01-15 1276 void __user *ukeys = u64_to_user_ptr(attr->batch.keys); 057996380a42bb Yonghong Song 2020-01-15 1277 void *ubatch = u64_to_user_ptr(attr->batch.in_batch); 057996380a42bb Yonghong Song 2020-01-15 1278 u32 batch, max_count, size, bucket_size; b9aff38de2cb16 Yonghong Song 2020-02-19 1279 struct htab_elem *node_to_free = NULL; 057996380a42bb Yonghong Song 2020-01-15 1280 u64 elem_map_flags, map_flags; 057996380a42bb Yonghong Song 2020-01-15 1281 struct hlist_nulls_head *head; 057996380a42bb Yonghong Song 2020-01-15 1282 struct hlist_nulls_node *n; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1283 unsigned long flags = 0; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1284 bool locked = false; 057996380a42bb Yonghong Song 2020-01-15 1285 struct htab_elem *l; 057996380a42bb Yonghong Song 2020-01-15 1286 struct bucket *b; 057996380a42bb Yonghong Song 2020-01-15 1287 int ret = 0; 057996380a42bb Yonghong Song 2020-01-15 1288 057996380a42bb Yonghong Song 2020-01-15 1289 elem_map_flags = attr->batch.elem_flags; 057996380a42bb Yonghong Song 2020-01-15 1290 if ((elem_map_flags & ~BPF_F_LOCK) || 057996380a42bb Yonghong Song 2020-01-15 1291 ((elem_map_flags & BPF_F_LOCK) && !map_value_has_spin_lock(map))) 057996380a42bb Yonghong Song 2020-01-15 1292 return -EINVAL; 057996380a42bb Yonghong Song 2020-01-15 1293 057996380a42bb Yonghong Song 2020-01-15 1294 map_flags = attr->batch.flags; 057996380a42bb Yonghong Song 2020-01-15 1295 if (map_flags) 057996380a42bb Yonghong Song 2020-01-15 1296 return -EINVAL; 057996380a42bb Yonghong Song 2020-01-15 1297 057996380a42bb Yonghong Song 2020-01-15 1298 max_count = attr->batch.count; 057996380a42bb Yonghong Song 2020-01-15 1299 if (!max_count) 057996380a42bb Yonghong Song 2020-01-15 1300 return 0; 057996380a42bb Yonghong Song 2020-01-15 1301 057996380a42bb Yonghong Song 2020-01-15 1302 if (put_user(0, &uattr->batch.count)) 057996380a42bb Yonghong Song 2020-01-15 1303 return -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1304 057996380a42bb Yonghong Song 2020-01-15 1305 batch = 0; 057996380a42bb Yonghong Song 2020-01-15 1306 if (ubatch && copy_from_user(&batch, ubatch, sizeof(batch))) 057996380a42bb Yonghong Song 2020-01-15 1307 return -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1308 057996380a42bb Yonghong Song 2020-01-15 1309 if (batch >= htab->n_buckets) This this use array_index_nospec() some how? 057996380a42bb Yonghong Song 2020-01-15 1310 return -ENOENT; 057996380a42bb Yonghong Song 2020-01-15 1311 057996380a42bb Yonghong Song 2020-01-15 1312 key_size = htab->map.key_size; 057996380a42bb Yonghong Song 2020-01-15 1313 roundup_key_size = round_up(htab->map.key_size, 8); 057996380a42bb Yonghong Song 2020-01-15 1314 value_size = htab->map.value_size; 057996380a42bb Yonghong Song 2020-01-15 1315 size = round_up(value_size, 8); 057996380a42bb Yonghong Song 2020-01-15 1316 if (is_percpu) 057996380a42bb Yonghong Song 2020-01-15 1317 value_size = size * num_possible_cpus(); 057996380a42bb Yonghong Song 2020-01-15 1318 total = 0; 057996380a42bb Yonghong Song 2020-01-15 1319 /* while experimenting with hash tables with sizes ranging from 10 to 057996380a42bb Yonghong Song 2020-01-15 1320 * 1000, it was observed that a bucket can have upto 5 entries. 057996380a42bb Yonghong Song 2020-01-15 1321 */ 057996380a42bb Yonghong Song 2020-01-15 1322 bucket_size = 5; 057996380a42bb Yonghong Song 2020-01-15 1323 057996380a42bb Yonghong Song 2020-01-15 1324 alloc: 057996380a42bb Yonghong Song 2020-01-15 1325 /* We cannot do copy_from_user or copy_to_user inside 057996380a42bb Yonghong Song 2020-01-15 1326 * the rcu_read_lock. Allocate enough space here. 057996380a42bb Yonghong Song 2020-01-15 1327 */ 057996380a42bb Yonghong Song 2020-01-15 1328 keys = kvmalloc(key_size * bucket_size, GFP_USER | __GFP_NOWARN); 057996380a42bb Yonghong Song 2020-01-15 1329 values = kvmalloc(value_size * bucket_size, GFP_USER | __GFP_NOWARN); 057996380a42bb Yonghong Song 2020-01-15 1330 if (!keys || !values) { 057996380a42bb Yonghong Song 2020-01-15 1331 ret = -ENOMEM; 057996380a42bb Yonghong Song 2020-01-15 1332 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1333 } 057996380a42bb Yonghong Song 2020-01-15 1334 057996380a42bb Yonghong Song 2020-01-15 1335 again: 085fee1a72a9fb Thomas Gleixner 2020-02-24 1336 bpf_disable_instrumentation(); 057996380a42bb Yonghong Song 2020-01-15 1337 rcu_read_lock(); 057996380a42bb Yonghong Song 2020-01-15 1338 again_nocopy: 057996380a42bb Yonghong Song 2020-01-15 1339 dst_key = keys; 057996380a42bb Yonghong Song 2020-01-15 1340 dst_val = values; 057996380a42bb Yonghong Song 2020-01-15 @1341 b = &htab->buckets[batch]; 057996380a42bb Yonghong Song 2020-01-15 1342 head = &b->head; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1343 /* do not grab the lock unless need it (bucket_cnt > 0). */ 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1344 if (locked) 057996380a42bb Yonghong Song 2020-01-15 1345 raw_spin_lock_irqsave(&b->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1346 057996380a42bb Yonghong Song 2020-01-15 1347 bucket_cnt = 0; 057996380a42bb Yonghong Song 2020-01-15 1348 hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) 057996380a42bb Yonghong Song 2020-01-15 1349 bucket_cnt++; 057996380a42bb Yonghong Song 2020-01-15 1350 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1351 if (bucket_cnt && !locked) { 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1352 locked = true; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1353 goto again_nocopy; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1354 } 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1355 057996380a42bb Yonghong Song 2020-01-15 1356 if (bucket_cnt > (max_count - total)) { 057996380a42bb Yonghong Song 2020-01-15 1357 if (total == 0) 057996380a42bb Yonghong Song 2020-01-15 1358 ret = -ENOSPC; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1359 /* Note that since bucket_cnt > 0 here, it is implicit 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1360 * that the locked was grabbed, so release it. 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1361 */ 057996380a42bb Yonghong Song 2020-01-15 1362 raw_spin_unlock_irqrestore(&b->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1363 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1364 bpf_enable_instrumentation(); 057996380a42bb Yonghong Song 2020-01-15 1365 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1366 } 057996380a42bb Yonghong Song 2020-01-15 1367 057996380a42bb Yonghong Song 2020-01-15 1368 if (bucket_cnt > bucket_size) { 057996380a42bb Yonghong Song 2020-01-15 1369 bucket_size = bucket_cnt; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1370 /* Note that since bucket_cnt > 0 here, it is implicit 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1371 * that the locked was grabbed, so release it. 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1372 */ 057996380a42bb Yonghong Song 2020-01-15 1373 raw_spin_unlock_irqrestore(&b->lock, flags); 057996380a42bb Yonghong Song 2020-01-15 1374 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1375 bpf_enable_instrumentation(); 057996380a42bb Yonghong Song 2020-01-15 1376 kvfree(keys); 057996380a42bb Yonghong Song 2020-01-15 1377 kvfree(values); 057996380a42bb Yonghong Song 2020-01-15 1378 goto alloc; 057996380a42bb Yonghong Song 2020-01-15 1379 } 057996380a42bb Yonghong Song 2020-01-15 1380 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1381 /* Next block is only safe to run if you have grabbed the lock */ 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1382 if (!locked) 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1383 goto next_batch; 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1384 057996380a42bb Yonghong Song 2020-01-15 1385 hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { 057996380a42bb Yonghong Song 2020-01-15 1386 memcpy(dst_key, l->key, key_size); 057996380a42bb Yonghong Song 2020-01-15 1387 057996380a42bb Yonghong Song 2020-01-15 1388 if (is_percpu) { 057996380a42bb Yonghong Song 2020-01-15 1389 int off = 0, cpu; 057996380a42bb Yonghong Song 2020-01-15 1390 void __percpu *pptr; 057996380a42bb Yonghong Song 2020-01-15 1391 057996380a42bb Yonghong Song 2020-01-15 1392 pptr = htab_elem_get_ptr(l, map->key_size); 057996380a42bb Yonghong Song 2020-01-15 1393 for_each_possible_cpu(cpu) { 057996380a42bb Yonghong Song 2020-01-15 1394 bpf_long_memcpy(dst_val + off, 057996380a42bb Yonghong Song 2020-01-15 1395 per_cpu_ptr(pptr, cpu), size); 057996380a42bb Yonghong Song 2020-01-15 1396 off += size; 057996380a42bb Yonghong Song 2020-01-15 1397 } 057996380a42bb Yonghong Song 2020-01-15 1398 } else { 057996380a42bb Yonghong Song 2020-01-15 1399 value = l->key + roundup_key_size; 057996380a42bb Yonghong Song 2020-01-15 1400 if (elem_map_flags & BPF_F_LOCK) 057996380a42bb Yonghong Song 2020-01-15 1401 copy_map_value_locked(map, dst_val, value, 057996380a42bb Yonghong Song 2020-01-15 1402 true); 057996380a42bb Yonghong Song 2020-01-15 1403 else 057996380a42bb Yonghong Song 2020-01-15 1404 copy_map_value(map, dst_val, value); 057996380a42bb Yonghong Song 2020-01-15 1405 check_and_init_map_lock(map, dst_val); 057996380a42bb Yonghong Song 2020-01-15 1406 } 057996380a42bb Yonghong Song 2020-01-15 1407 if (do_delete) { 057996380a42bb Yonghong Song 2020-01-15 1408 hlist_nulls_del_rcu(&l->hash_node); b9aff38de2cb16 Yonghong Song 2020-02-19 1409 b9aff38de2cb16 Yonghong Song 2020-02-19 1410 /* bpf_lru_push_free() will acquire lru_lock, which b9aff38de2cb16 Yonghong Song 2020-02-19 1411 * may cause deadlock. See comments in function b9aff38de2cb16 Yonghong Song 2020-02-19 1412 * prealloc_lru_pop(). Let us do bpf_lru_push_free() b9aff38de2cb16 Yonghong Song 2020-02-19 1413 * after releasing the bucket lock. b9aff38de2cb16 Yonghong Song 2020-02-19 1414 */ b9aff38de2cb16 Yonghong Song 2020-02-19 1415 if (is_lru_map) { b9aff38de2cb16 Yonghong Song 2020-02-19 1416 l->batch_flink = node_to_free; b9aff38de2cb16 Yonghong Song 2020-02-19 1417 node_to_free = l; b9aff38de2cb16 Yonghong Song 2020-02-19 1418 } else { 057996380a42bb Yonghong Song 2020-01-15 1419 free_htab_elem(htab, l); 057996380a42bb Yonghong Song 2020-01-15 1420 } b9aff38de2cb16 Yonghong Song 2020-02-19 1421 } 057996380a42bb Yonghong Song 2020-01-15 1422 dst_key += key_size; 057996380a42bb Yonghong Song 2020-01-15 1423 dst_val += value_size; 057996380a42bb Yonghong Song 2020-01-15 1424 } 057996380a42bb Yonghong Song 2020-01-15 1425 057996380a42bb Yonghong Song 2020-01-15 1426 raw_spin_unlock_irqrestore(&b->lock, flags); 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1427 locked = false; b9aff38de2cb16 Yonghong Song 2020-02-19 1428 b9aff38de2cb16 Yonghong Song 2020-02-19 1429 while (node_to_free) { b9aff38de2cb16 Yonghong Song 2020-02-19 1430 l = node_to_free; b9aff38de2cb16 Yonghong Song 2020-02-19 1431 node_to_free = node_to_free->batch_flink; b9aff38de2cb16 Yonghong Song 2020-02-19 1432 bpf_lru_push_free(&htab->lru, &l->lru_node); b9aff38de2cb16 Yonghong Song 2020-02-19 1433 } b9aff38de2cb16 Yonghong Song 2020-02-19 1434 492e0d0d6f2eb4 Brian Vazquez 2020-02-18 1435 next_batch: 057996380a42bb Yonghong Song 2020-01-15 1436 /* If we are not copying data, we can go to next bucket and avoid 057996380a42bb Yonghong Song 2020-01-15 1437 * unlocking the rcu. 057996380a42bb Yonghong Song 2020-01-15 1438 */ 057996380a42bb Yonghong Song 2020-01-15 1439 if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { 057996380a42bb Yonghong Song 2020-01-15 1440 batch++; 057996380a42bb Yonghong Song 2020-01-15 1441 goto again_nocopy; 057996380a42bb Yonghong Song 2020-01-15 1442 } 057996380a42bb Yonghong Song 2020-01-15 1443 057996380a42bb Yonghong Song 2020-01-15 1444 rcu_read_unlock(); 085fee1a72a9fb Thomas Gleixner 2020-02-24 1445 bpf_enable_instrumentation(); 057996380a42bb Yonghong Song 2020-01-15 1446 if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys, 057996380a42bb Yonghong Song 2020-01-15 1447 key_size * bucket_cnt) || 057996380a42bb Yonghong Song 2020-01-15 1448 copy_to_user(uvalues + total * value_size, values, 057996380a42bb Yonghong Song 2020-01-15 1449 value_size * bucket_cnt))) { 057996380a42bb Yonghong Song 2020-01-15 1450 ret = -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1451 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1452 } 057996380a42bb Yonghong Song 2020-01-15 1453 057996380a42bb Yonghong Song 2020-01-15 1454 total += bucket_cnt; 057996380a42bb Yonghong Song 2020-01-15 1455 batch++; 057996380a42bb Yonghong Song 2020-01-15 1456 if (batch >= htab->n_buckets) { 057996380a42bb Yonghong Song 2020-01-15 1457 ret = -ENOENT; 057996380a42bb Yonghong Song 2020-01-15 1458 goto after_loop; 057996380a42bb Yonghong Song 2020-01-15 1459 } 057996380a42bb Yonghong Song 2020-01-15 1460 goto again; 057996380a42bb Yonghong Song 2020-01-15 1461 057996380a42bb Yonghong Song 2020-01-15 1462 after_loop: 057996380a42bb Yonghong Song 2020-01-15 1463 if (ret == -EFAULT) 057996380a42bb Yonghong Song 2020-01-15 1464 goto out; 057996380a42bb Yonghong Song 2020-01-15 1465 057996380a42bb Yonghong Song 2020-01-15 1466 /* copy # of entries and next batch */ 057996380a42bb Yonghong Song 2020-01-15 1467 ubatch = u64_to_user_ptr(attr->batch.out_batch); 057996380a42bb Yonghong Song 2020-01-15 1468 if (copy_to_user(ubatch, &batch, sizeof(batch)) || 057996380a42bb Yonghong Song 2020-01-15 1469 put_user(total, &uattr->batch.count)) 057996380a42bb Yonghong Song 2020-01-15 1470 ret = -EFAULT; 057996380a42bb Yonghong Song 2020-01-15 1471 057996380a42bb Yonghong Song 2020-01-15 1472 out: 057996380a42bb Yonghong Song 2020-01-15 1473 kvfree(keys); 057996380a42bb Yonghong Song 2020-01-15 1474 kvfree(values); 057996380a42bb Yonghong Song 2020-01-15 1475 return ret; 057996380a42bb Yonghong Song 2020-01-15 1476 } --- 0-DAY CI Kernel Test Service, Intel Corporation https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org