[{"data":1,"prerenderedAt":2861},["ShallowReactive",2],{"project-lt-two-layers-neural-networks":3},{"id":4,"title":5,"description":6,"extension":7,"favorite":8,"icon":9,"meta":10,"publishedAt":2848,"readingTime":2849,"shortDescription":2850,"slug":2851,"status":2852,"stem":2853,"tags":2854,"type":2859,"__hash__":2860},"projects\u002Fprojects\u002Flt-two-layers-neural-networks.md","Generalization Bounds for Two-Layer Neural Networks with ReLU","A rigorous mathematical examination of generalization bounds for two-layer neural networks with ReLU activation. This project derives tight, width-independent complexity bounds by exploiting the scale-invariant properties of ReLU networks, moving beyond traditional empirical Rademacher complexity measures that fail for overparameterized models.","md",false,"i-ph-function-duotone",{"body":11},{"type":12,"value":13,"toc":2837},"minimark",[14,19,23,207,211,527,874,878,1299,1386,1390,1571,1764,1767,1771,2508,2759,2791,2795,2827,2831],[15,16,18],"h2",{"id":17},"overview","Overview",[20,21,22],"p",{},"This rigorous mathematical exercise derives and analyzes generalization bounds for two-layer neural networks with ReLU activation, progressively building from naive bounds to tight, width-independent complexity measures:",[24,25,26,34,195,201],"ul",{},[27,28,29,33],"li",{},[30,31,32],"strong",{},"Naive bound",": Classical empirical Rademacher complexity approach (demonstrates why standard bounds fail for overparameterized models)",[27,35,36,39,40,194],{},[30,37,38],{},"Symmetrization inequality",": Specialized for ReLU networks, exploiting ",[41,42,45,104],"span",{"className":43},[44],"katex",[41,46,49],{"className":47},[48],"katex-mathml",[50,51,53],"math",{"xmlns":52},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[54,55,56,99],"semantics",{},[57,58,59,64,67,69,73,76,80,82,85,88,90,92,95,97],"mrow",{},[60,61,63],"mi",{"mathvariant":62},"normal","∣",[60,65,66],{},"z",[60,68,63],{"mathvariant":62},[70,71,72],"mo",{},"=",[60,74,75],{},"ϕ",[70,77,79],{"stretchy":78},"false","(",[60,81,66],{},[70,83,84],{"stretchy":78},")",[70,86,87],{},"+",[60,89,75],{},[70,91,79],{"stretchy":78},[70,93,94],{},"−",[60,96,66],{},[70,98,84],{"stretchy":78},[100,101,103],"annotation",{"encoding":102},"application\u002Fx-tex","|z| = \\phi(z) + \\phi(-z)",[41,105,109,142,173],{"className":106,"ariaHidden":108},[107],"katex-html","true",[41,110,113,118,122,127,130,135,139],{"className":111},[112],"base",[41,114],{"className":115,"style":117},[116],"strut","height:1em;vertical-align:-0.25em;",[41,119,63],{"className":120},[121],"mord",[41,123,66],{"className":124,"style":126},[121,125],"mathnormal","margin-right:0.044em;",[41,128,63],{"className":129},[121],[41,131],{"className":132,"style":134},[133],"mspace","margin-right:0.2778em;",[41,136,72],{"className":137},[138],"mrel",[41,140],{"className":141,"style":134},[133],[41,143,145,148,151,155,158,162,166,170],{"className":144},[112],[41,146],{"className":147,"style":117},[116],[41,149,75],{"className":150},[121,125],[41,152,79],{"className":153},[154],"mopen",[41,156,66],{"className":157,"style":126},[121,125],[41,159,84],{"className":160},[161],"mclose",[41,163],{"className":164,"style":165},[133],"margin-right:0.2222em;",[41,167,87],{"className":168},[169],"mbin",[41,171],{"className":172,"style":165},[133],[41,174,176,179,182,185,188,191],{"className":175},[112],[41,177],{"className":178,"style":117},[116],[41,180,75],{"className":181},[121,125],[41,183,79],{"className":184},[154],[41,186,94],{"className":187},[121],[41,189,66],{"className":190,"style":126},[121,125],[41,192,84],{"className":193},[161]," structure",[27,196,197,200],{},[30,198,199],{},"Scale-invariant complexity measure",": Leverages ReLU positive homogeneity property",[27,202,203,206],{},[30,204,205],{},"Width-independent bound",": Final tight generalization bound that holds regardless of network width",[15,208,210],{"id":209},"theoretical-foundation","Theoretical Foundation",[20,212,213,214],{},"Two-layer neural networks computing functions of the form:\n",[41,215,217,288],{"className":216},[44],[41,218,220],{"className":219},[48],[50,221,222],{"xmlns":52},[54,223,224,285],{},[57,225,226,229,231,234,236,238,258,266,269,271,281,283],{},[60,227,228],{},"f",[70,230,79],{"stretchy":78},[60,232,233],{},"x",[70,235,84],{"stretchy":78},[70,237,72],{},[239,240,241,244,255],"msubsup",{},[70,242,243],{},"∑",[57,245,246,249,251],{},[60,247,248],{},"j",[70,250,72],{},[252,253,254],"mn",{},"1",[60,256,257],{},"m",[259,260,261,264],"msub",{},[60,262,263],{},"u",[60,265,248],{},[60,267,268],{},"σ",[70,270,79],{"stretchy":78},[239,272,273,276,278],{},[60,274,275],{},"w",[60,277,248],{},[60,279,280],{"mathvariant":62},"⊤",[60,282,233],{},[70,284,84],{"stretchy":78},[100,286,287],{"encoding":102},"f(x) = \\sum_{j=1}^{m} u_j \\sigma(w_j^\\top x)",[41,289,291,319],{"className":290,"ariaHidden":108},[107],[41,292,294,297,301,304,307,310,313,316],{"className":293},[112],[41,295],{"className":296,"style":117},[116],[41,298,228],{"className":299,"style":300},[121,125],"margin-right:0.1076em;",[41,302,79],{"className":303},[154],[41,305,233],{"className":306},[121,125],[41,308,84],{"className":309},[161],[41,311],{"className":312,"style":134},[133],[41,314,72],{"className":315},[138],[41,317],{"className":318,"style":134},[133],[41,320,322,326,411,415,458,462,465,521,524],{"className":321},[112],[41,323],{"className":324,"style":325},[116],"height:1.2849em;vertical-align:-0.4358em;",[41,327,330,336],{"className":328},[329],"mop",[41,331,243],{"className":332,"style":335},[329,333,334],"op-symbol","small-op","position:relative;top:0em;",[41,337,340],{"className":338},[339],"msupsub",[41,341,345,402],{"className":342},[343,344],"vlist-t","vlist-t2",[41,346,349,397],{"className":347},[348],"vlist-r",[41,350,354,382],{"className":351,"style":353},[352],"vlist","height:0.8043em;",[41,355,357,362],{"style":356},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[41,358],{"className":359,"style":361},[360],"pstrut","height:2.7em;",[41,363,369],{"className":364},[365,366,367,368],"sizing","reset-size6","size3","mtight",[41,370,372,376,379],{"className":371},[121,368],[41,373,248],{"className":374,"style":375},[121,125,368],"margin-right:0.0572em;",[41,377,72],{"className":378},[138,368],[41,380,254],{"className":381},[121,368],[41,383,385,388],{"style":384},"top:-3.2029em;margin-right:0.05em;",[41,386],{"className":387,"style":361},[360],[41,389,391],{"className":390},[365,366,367,368],[41,392,394],{"className":393},[121,368],[41,395,257],{"className":396},[121,125,368],[41,398,401],{"className":399},[400],"vlist-s","​",[41,403,405],{"className":404},[348],[41,406,409],{"className":407,"style":408},[352],"height:0.4358em;",[41,410],{},[41,412],{"className":413,"style":414},[133],"margin-right:0.1667em;",[41,416,418,421],{"className":417},[121],[41,419,263],{"className":420},[121,125],[41,422,424],{"className":423},[339],[41,425,427,449],{"className":426},[343,344],[41,428,430,446],{"className":429},[348],[41,431,434],{"className":432,"style":433},[352],"height:0.3117em;",[41,435,437,440],{"style":436},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[41,438],{"className":439,"style":361},[360],[41,441,443],{"className":442},[365,366,367,368],[41,444,248],{"className":445,"style":375},[121,125,368],[41,447,401],{"className":448},[400],[41,450,452],{"className":451},[348],[41,453,456],{"className":454,"style":455},[352],"height:0.2861em;",[41,457],{},[41,459,268],{"className":460,"style":461},[121,125],"margin-right:0.0359em;",[41,463,79],{"className":464},[154],[41,466,468,472],{"className":467},[121],[41,469,275],{"className":470,"style":471},[121,125],"margin-right:0.0269em;",[41,473,475],{"className":474},[339],[41,476,478,512],{"className":477},[343,344],[41,479,481,509],{"className":480},[348],[41,482,485,497],{"className":483,"style":484},[352],"height:0.8491em;",[41,486,488,491],{"style":487},"top:-2.4413em;margin-left:-0.0269em;margin-right:0.05em;",[41,489],{"className":490,"style":361},[360],[41,492,494],{"className":493},[365,366,367,368],[41,495,248],{"className":496,"style":375},[121,125,368],[41,498,500,503],{"style":499},"top:-3.063em;margin-right:0.05em;",[41,501],{"className":502,"style":361},[360],[41,504,506],{"className":505},[365,366,367,368],[41,507,280],{"className":508},[121,368],[41,510,401],{"className":511},[400],[41,513,515],{"className":514},[348],[41,516,519],{"className":517,"style":518},[352],"height:0.3948em;",[41,520],{},[41,522,233],{"className":523},[121,125],[41,525,84],{"className":526},[161],[20,528,529,530,575,576,729,730,873],{},"where ",[41,531,533,554],{"className":532},[44],[41,534,536],{"className":535},[48],[50,537,538],{"xmlns":52},[54,539,540,551],{},[57,541,542,544,546,549],{},[60,543,268],{},[70,545,79],{"stretchy":78},[70,547,548],{},"⋅",[70,550,84],{"stretchy":78},[100,552,553],{"encoding":102},"\\sigma(\\cdot)",[41,555,557],{"className":556,"ariaHidden":108},[107],[41,558,560,563,566,569,572],{"className":559},[112],[41,561],{"className":562,"style":117},[116],[41,564,268],{"className":565,"style":461},[121,125],[41,567,79],{"className":568},[154],[41,570,548],{"className":571},[121],[41,573,84],{"className":574},[161]," is the ReLU activation function and parameters are bounded by ",[41,577,579,612],{"className":578},[44],[41,580,582],{"className":581},[48],[50,583,584],{"xmlns":52},[54,585,586,609],{},[57,587,588,591,597,599,602],{},[60,589,590],{"mathvariant":62},"∥",[259,592,593,595],{},[60,594,275],{},[60,596,248],{},[60,598,590],{"mathvariant":62},[70,600,601],{},"≤",[259,603,604,607],{},[60,605,606],{},"B",[60,608,275],{},[100,610,611],{"encoding":102},"\\|w_j\\| \\leq B_w",[41,613,615,678],{"className":614,"ariaHidden":108},[107],[41,616,618,622,625,666,669,672,675],{"className":617},[112],[41,619],{"className":620,"style":621},[116],"height:1.0361em;vertical-align:-0.2861em;",[41,623,590],{"className":624},[121],[41,626,628,631],{"className":627},[121],[41,629,275],{"className":630,"style":471},[121,125],[41,632,634],{"className":633},[339],[41,635,637,658],{"className":636},[343,344],[41,638,640,655],{"className":639},[348],[41,641,643],{"className":642,"style":433},[352],[41,644,646,649],{"style":645},"top:-2.55em;margin-left:-0.0269em;margin-right:0.05em;",[41,647],{"className":648,"style":361},[360],[41,650,652],{"className":651},[365,366,367,368],[41,653,248],{"className":654,"style":375},[121,125,368],[41,656,401],{"className":657},[400],[41,659,661],{"className":660},[348],[41,662,664],{"className":663,"style":455},[352],[41,665],{},[41,667,590],{"className":668},[121],[41,670],{"className":671,"style":134},[133],[41,673,601],{"className":674},[138],[41,676],{"className":677,"style":134},[133],[41,679,681,685],{"className":680},[112],[41,682],{"className":683,"style":684},[116],"height:0.8333em;vertical-align:-0.15em;",[41,686,688,692],{"className":687},[121],[41,689,606],{"className":690,"style":691},[121,125],"margin-right:0.0502em;",[41,693,695],{"className":694},[339],[41,696,698,720],{"className":697},[343,344],[41,699,701,717],{"className":700},[348],[41,702,705],{"className":703,"style":704},[352],"height:0.1514em;",[41,706,708,711],{"style":707},"top:-2.55em;margin-left:-0.0502em;margin-right:0.05em;",[41,709],{"className":710,"style":361},[360],[41,712,714],{"className":713},[365,366,367,368],[41,715,275],{"className":716,"style":471},[121,125,368],[41,718,401],{"className":719},[400],[41,721,723],{"className":722},[348],[41,724,727],{"className":725,"style":726},[352],"height:0.15em;",[41,728],{}," and ",[41,731,733,763],{"className":732},[44],[41,734,736],{"className":735},[48],[50,737,738],{"xmlns":52},[54,739,740,760],{},[57,741,742,744,750,752,754],{},[60,743,63],{"mathvariant":62},[259,745,746,748],{},[60,747,263],{},[60,749,248],{},[60,751,63],{"mathvariant":62},[70,753,601],{},[259,755,756,758],{},[60,757,606],{},[60,759,263],{},[100,761,762],{"encoding":102},"|u_j| \\leq B_u",[41,764,766,827],{"className":765,"ariaHidden":108},[107],[41,767,769,772,775,815,818,821,824],{"className":768},[112],[41,770],{"className":771,"style":621},[116],[41,773,63],{"className":774},[121],[41,776,778,781],{"className":777},[121],[41,779,263],{"className":780},[121,125],[41,782,784],{"className":783},[339],[41,785,787,807],{"className":786},[343,344],[41,788,790,804],{"className":789},[348],[41,791,793],{"className":792,"style":433},[352],[41,794,795,798],{"style":436},[41,796],{"className":797,"style":361},[360],[41,799,801],{"className":800},[365,366,367,368],[41,802,248],{"className":803,"style":375},[121,125,368],[41,805,401],{"className":806},[400],[41,808,810],{"className":809},[348],[41,811,813],{"className":812,"style":455},[352],[41,814],{},[41,816,63],{"className":817},[121],[41,819],{"className":820,"style":134},[133],[41,822,601],{"className":823},[138],[41,825],{"className":826,"style":134},[133],[41,828,830,833],{"className":829},[112],[41,831],{"className":832,"style":684},[116],[41,834,836,839],{"className":835},[121],[41,837,606],{"className":838,"style":691},[121,125],[41,840,842],{"className":841},[339],[41,843,845,865],{"className":844},[343,344],[41,846,848,862],{"className":847},[348],[41,849,851],{"className":850,"style":704},[352],[41,852,853,856],{"style":707},[41,854],{"className":855,"style":361},[360],[41,857,859],{"className":858},[365,366,367,368],[41,860,263],{"className":861},[121,125,368],[41,863,401],{"className":864},[400],[41,866,868],{"className":867},[348],[41,869,871],{"className":870,"style":726},[352],[41,872],{},".",[15,875,877],{"id":876},"part-a-naive-width-dependent-bound","Part A: Naive Width-Dependent Bound",[20,879,880,881],{},"Starting with standard empirical Rademacher complexity analysis: ",[41,882,884,947],{"className":883},[44],[41,885,887],{"className":886},[48],[50,888,889],{"xmlns":52},[54,890,891,944],{},[57,892,893,907,909,912,914,916,919,925,931,934],{},[259,894,895,899],{},[60,896,898],{"mathvariant":897},"script","R",[259,900,901,904],{},[60,902,903],{},"S",[60,905,906],{},"n",[70,908,79],{"stretchy":78},[60,910,911],{"mathvariant":897},"H",[70,913,84],{"stretchy":78},[70,915,601],{},[252,917,918],{},"2",[259,920,921,923],{},[60,922,606],{},[60,924,275],{},[259,926,927,929],{},[60,928,606],{},[60,930,263],{},[60,932,933],{},"C",[935,936,937],"msqrt",{},[938,939,940,942],"mfrac",{},[60,941,257],{},[60,943,906],{},[100,945,946],{"encoding":102},"\\mathcal{R}_{S_n}(\\mathcal{H}) \\leq 2 B_w B_u C \\sqrt{\\frac{m}{n}}",[41,948,950,1066],{"className":949,"ariaHidden":108},[107],[41,951,953,957,1047,1050,1054,1057,1060,1063],{"className":952},[112],[41,954],{"className":955,"style":956},[116],"height:1.0001em;vertical-align:-0.2501em;",[41,958,960,964],{"className":959},[121],[41,961,898],{"className":962},[121,963],"mathcal",[41,965,967],{"className":966},[339],[41,968,970,1038],{"className":969},[343,344],[41,971,973,1035],{"className":972},[348],[41,974,977],{"className":975,"style":976},[352],"height:0.3283em;",[41,978,979,982],{"style":436},[41,980],{"className":981,"style":361},[360],[41,983,985],{"className":984},[365,366,367,368],[41,986,988],{"className":987},[121,368],[41,989,991,995],{"className":990},[121,368],[41,992,903],{"className":993,"style":994},[121,125,368],"margin-right:0.0576em;",[41,996,998],{"className":997},[339],[41,999,1001,1026],{"className":1000},[343,344],[41,1002,1004,1023],{"className":1003},[348],[41,1005,1008],{"className":1006,"style":1007},[352],"height:0.1645em;",[41,1009,1011,1015],{"style":1010},"top:-2.357em;margin-left:-0.0576em;margin-right:0.0714em;",[41,1012],{"className":1013,"style":1014},[360],"height:2.5em;",[41,1016,1020],{"className":1017},[365,1018,1019,368],"reset-size3","size1",[41,1021,906],{"className":1022},[121,125,368],[41,1024,401],{"className":1025},[400],[41,1027,1029],{"className":1028},[348],[41,1030,1033],{"className":1031,"style":1032},[352],"height:0.143em;",[41,1034],{},[41,1036,401],{"className":1037},[400],[41,1039,1041],{"className":1040},[348],[41,1042,1045],{"className":1043,"style":1044},[352],"height:0.2501em;",[41,1046],{},[41,1048,79],{"className":1049},[154],[41,1051,911],{"className":1052,"style":1053},[121,963],"margin-right:0.0097em;",[41,1055,84],{"className":1056},[161],[41,1058],{"className":1059,"style":134},[133],[41,1061,601],{"className":1062},[138],[41,1064],{"className":1065,"style":134},[133],[41,1067,1069,1073,1076,1116,1156,1160],{"className":1068},[112],[41,1070],{"className":1071,"style":1072},[116],"height:1.24em;vertical-align:-0.3798em;",[41,1074,918],{"className":1075},[121],[41,1077,1079,1082],{"className":1078},[121],[41,1080,606],{"className":1081,"style":691},[121,125],[41,1083,1085],{"className":1084},[339],[41,1086,1088,1108],{"className":1087},[343,344],[41,1089,1091,1105],{"className":1090},[348],[41,1092,1094],{"className":1093,"style":704},[352],[41,1095,1096,1099],{"style":707},[41,1097],{"className":1098,"style":361},[360],[41,1100,1102],{"className":1101},[365,366,367,368],[41,1103,275],{"className":1104,"style":471},[121,125,368],[41,1106,401],{"className":1107},[400],[41,1109,1111],{"className":1110},[348],[41,1112,1114],{"className":1113,"style":726},[352],[41,1115],{},[41,1117,1119,1122],{"className":1118},[121],[41,1120,606],{"className":1121,"style":691},[121,125],[41,1123,1125],{"className":1124},[339],[41,1126,1128,1148],{"className":1127},[343,344],[41,1129,1131,1145],{"className":1130},[348],[41,1132,1134],{"className":1133,"style":704},[352],[41,1135,1136,1139],{"style":707},[41,1137],{"className":1138,"style":361},[360],[41,1140,1142],{"className":1141},[365,366,367,368],[41,1143,263],{"className":1144},[121,125,368],[41,1146,401],{"className":1147},[400],[41,1149,1151],{"className":1150},[348],[41,1152,1154],{"className":1153,"style":726},[352],[41,1155],{},[41,1157,933],{"className":1158,"style":1159},[121,125],"margin-right:0.0715em;",[41,1161,1164],{"className":1162},[121,1163],"sqrt",[41,1165,1167,1290],{"className":1166},[343,344],[41,1168,1170,1287],{"className":1169},[348],[41,1171,1174,1264],{"className":1172,"style":1173},[352],"height:0.8602em;",[41,1175,1179,1183],{"className":1176,"style":1178},[1177],"svg-align","top:-3.2em;",[41,1180],{"className":1181,"style":1182},[360],"height:3.2em;",[41,1184,1187],{"className":1185,"style":1186},[121],"padding-left:1em;",[41,1188,1190,1194,1261],{"className":1189},[121],[41,1191],{"className":1192},[154,1193],"nulldelimiter",[41,1195,1197],{"className":1196},[938],[41,1198,1200,1252],{"className":1199},[343,344],[41,1201,1203,1249],{"className":1202},[348],[41,1204,1207,1223,1234],{"className":1205,"style":1206},[352],"height:0.6954em;",[41,1208,1210,1214],{"style":1209},"top:-2.655em;",[41,1211],{"className":1212,"style":1213},[360],"height:3em;",[41,1215,1217],{"className":1216},[365,366,367,368],[41,1218,1220],{"className":1219},[121,368],[41,1221,906],{"className":1222},[121,125,368],[41,1224,1226,1229],{"style":1225},"top:-3.23em;",[41,1227],{"className":1228,"style":1213},[360],[41,1230],{"className":1231,"style":1233},[1232],"frac-line","border-bottom-width:0.04em;",[41,1235,1237,1240],{"style":1236},"top:-3.394em;",[41,1238],{"className":1239,"style":1213},[360],[41,1241,1243],{"className":1242},[365,366,367,368],[41,1244,1246],{"className":1245},[121,368],[41,1247,257],{"className":1248},[121,125,368],[41,1250,401],{"className":1251},[400],[41,1253,1255],{"className":1254},[348],[41,1256,1259],{"className":1257,"style":1258},[352],"height:0.345em;",[41,1260],{},[41,1262],{"className":1263},[161,1193],[41,1265,1267,1270],{"style":1266},"top:-2.8202em;",[41,1268],{"className":1269,"style":1182},[360],[41,1271,1275],{"className":1272,"style":1274},[1273],"hide-tail","min-width:1.02em;height:1.28em;",[1276,1277,1283],"svg",{"xmlns":1278,"width":1279,"height":1280,"viewBox":1281,"preserveAspectRatio":1282},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.28em","0 0 400000 1296","xMinYMin slice",[1284,1285],"path",{"d":1286},"M263,681c0.7,0,18,39.7,52,119\nc34,79.3,68.167,158.7,102.5,238c34.3,79.3,51.8,119.3,52.5,120\nc340,-704.7,510.7,-1060.3,512,-1067\nl0 -0\nc4.7,-7.3,11,-11,19,-11\nH40000v40H1012.3\ns-271.3,567,-271.3,567c-38.7,80.7,-84,175,-136,283c-52,108,-89.167,185.3,-111.5,232\nc-22.3,46.7,-33.8,70.3,-34.5,71c-4.7,4.7,-12.3,7,-23,7s-12,-1,-12,-1\ns-109,-253,-109,-253c-72.7,-168,-109.3,-252,-110,-252c-10.7,8,-22,16.7,-34,26\nc-22,17.3,-33.3,26,-34,26s-26,-26,-26,-26s76,-59,76,-59s76,-60,76,-60z\nM1001 80h400000v40h-400000z",[41,1288,401],{"className":1289},[400],[41,1291,1293],{"className":1292},[348],[41,1294,1297],{"className":1295,"style":1296},[352],"height:0.3798em;",[41,1298],{},[20,1300,1301,1302,1331,1332,1385],{},"This bound demonstrates the fundamental problem: it depends linearly on network width ",[41,1303,1305,1318],{"className":1304},[44],[41,1306,1308],{"className":1307},[48],[50,1309,1310],{"xmlns":52},[54,1311,1312,1316],{},[57,1313,1314],{},[60,1315,257],{},[100,1317,257],{"encoding":102},[41,1319,1321],{"className":1320,"ariaHidden":108},[107],[41,1322,1324,1328],{"className":1323},[112],[41,1325],{"className":1326,"style":1327},[116],"height:0.4306em;",[41,1329,257],{"className":1330},[121,125],", making it vacuous for overparameterized networks where ",[41,1333,1335,1354],{"className":1334},[44],[41,1336,1338],{"className":1337},[48],[50,1339,1340],{"xmlns":52},[54,1341,1342,1351],{},[57,1343,1344,1346,1349],{},[60,1345,257],{},[70,1347,1348],{},"≫",[60,1350,906],{},[100,1352,1353],{"encoding":102},"m \\gg n",[41,1355,1357,1376],{"className":1356,"ariaHidden":108},[107],[41,1358,1360,1364,1367,1370,1373],{"className":1359},[112],[41,1361],{"className":1362,"style":1363},[116],"height:0.5782em;vertical-align:-0.0391em;",[41,1365,257],{"className":1366},[121,125],[41,1368],{"className":1369,"style":134},[133],[41,1371,1348],{"className":1372},[138],[41,1374],{"className":1375,"style":134},[133],[41,1377,1379,1382],{"className":1378},[112],[41,1380],{"className":1381,"style":1327},[116],[41,1383,906],{"className":1384},[121,125],". This gap between theory and practice motivated the subsequent refinements.",[15,1387,1389],{"id":1388},"part-b-symmetrization-inequality-for-relu","Part B: Symmetrization Inequality for ReLU",[20,1391,1392,1393,1570],{},"Key insight exploiting ReLU structure: ",[41,1394,1396,1459],{"className":1395},[44],[41,1397,1399],{"className":1398},[48],[50,1400,1401],{"xmlns":52},[54,1402,1403,1456],{},[57,1404,1405,1409,1430,1432,1434,1436,1438],{},[60,1406,1408],{"mathvariant":1407},"double-struck","E",[57,1410,1411,1414,1417,1420,1422,1425,1427],{},[70,1412,1413],{"fence":108},"[",[60,1415,1416],{},"sup",[70,1418,1419],{},"⁡",[60,1421,63],{"mathvariant":62},[60,1423,1424],{"mathvariant":897},"Z",[60,1426,63],{"mathvariant":62},[70,1428,1429],{"fence":108},"]",[70,1431,601],{},[252,1433,918],{},[70,1435,548],{},[60,1437,1408],{"mathvariant":1407},[57,1439,1440,1442,1444,1446,1448,1450,1452,1454],{},[70,1441,1413],{"fence":108},[60,1443,1416],{},[70,1445,1419],{},[60,1447,75],{},[70,1449,79],{"stretchy":78},[60,1451,1424],{"mathvariant":897},[70,1453,84],{"stretchy":78},[70,1455,1429],{"fence":108},[100,1457,1458],{"encoding":102},"\\mathbb{E}\\left[\\sup |\\mathcal{Z}|\\right] \\leq 2 \\cdot \\mathbb{E}\\left[\\sup \\phi(\\mathcal{Z})\\right]",[41,1460,1462,1512,1531],{"className":1461,"ariaHidden":108},[107],[41,1463,1465,1468,1472,1475,1503,1506,1509],{"className":1464},[112],[41,1466],{"className":1467,"style":117},[116],[41,1469,1408],{"className":1470},[121,1471],"mathbb",[41,1473],{"className":1474,"style":414},[133],[41,1476,1479,1484,1487,1490,1493,1497,1500],{"className":1477},[1478],"minner",[41,1480,1413],{"className":1481,"style":1483},[154,1482],"delimcenter","top:0em;",[41,1485,1416],{"className":1486},[329],[41,1488],{"className":1489,"style":414},[133],[41,1491,63],{"className":1492},[121],[41,1494,1424],{"className":1495,"style":1496},[121,963],"margin-right:0.0794em;",[41,1498,63],{"className":1499},[121],[41,1501,1429],{"className":1502,"style":1483},[161,1482],[41,1504],{"className":1505,"style":134},[133],[41,1507,601],{"className":1508},[138],[41,1510],{"className":1511,"style":134},[133],[41,1513,1515,1519,1522,1525,1528],{"className":1514},[112],[41,1516],{"className":1517,"style":1518},[116],"height:0.6444em;",[41,1520,918],{"className":1521},[121],[41,1523],{"className":1524,"style":165},[133],[41,1526,548],{"className":1527},[169],[41,1529],{"className":1530,"style":165},[133],[41,1532,1534,1537,1540,1543],{"className":1533},[112],[41,1535],{"className":1536,"style":117},[116],[41,1538,1408],{"className":1539},[121,1471],[41,1541],{"className":1542,"style":414},[133],[41,1544,1546,1549,1552,1555,1558,1561,1564,1567],{"className":1545},[1478],[41,1547,1413],{"className":1548,"style":1483},[154,1482],[41,1550,1416],{"className":1551},[329],[41,1553],{"className":1554,"style":414},[133],[41,1556,75],{"className":1557},[121,125],[41,1559,79],{"className":1560},[154],[41,1562,1424],{"className":1563,"style":1496},[121,963],[41,1565,84],{"className":1566},[161],[41,1568,1429],{"className":1569,"style":1483},[161,1482],"\nDerivation uses:",[24,1572,1573,1693,1761],{},[27,1574,1575,1576],{},"ReLU decomposition: ",[41,1577,1579,1618],{"className":1578},[44],[41,1580,1582],{"className":1581},[48],[50,1583,1584],{"xmlns":52},[54,1585,1586,1616],{},[57,1587,1588,1590,1592,1594,1596,1598,1600,1602,1604,1606,1608,1610,1612,1614],{},[60,1589,63],{"mathvariant":62},[60,1591,66],{},[60,1593,63],{"mathvariant":62},[70,1595,72],{},[60,1597,75],{},[70,1599,79],{"stretchy":78},[60,1601,66],{},[70,1603,84],{"stretchy":78},[70,1605,87],{},[60,1607,75],{},[70,1609,79],{"stretchy":78},[70,1611,94],{},[60,1613,66],{},[70,1615,84],{"stretchy":78},[100,1617,103],{"encoding":102},[41,1619,1621,1645,1672],{"className":1620,"ariaHidden":108},[107],[41,1622,1624,1627,1630,1633,1636,1639,1642],{"className":1623},[112],[41,1625],{"className":1626,"style":117},[116],[41,1628,63],{"className":1629},[121],[41,1631,66],{"className":1632,"style":126},[121,125],[41,1634,63],{"className":1635},[121],[41,1637],{"className":1638,"style":134},[133],[41,1640,72],{"className":1641},[138],[41,1643],{"className":1644,"style":134},[133],[41,1646,1648,1651,1654,1657,1660,1663,1666,1669],{"className":1647},[112],[41,1649],{"className":1650,"style":117},[116],[41,1652,75],{"className":1653},[121,125],[41,1655,79],{"className":1656},[154],[41,1658,66],{"className":1659,"style":126},[121,125],[41,1661,84],{"className":1662},[161],[41,1664],{"className":1665,"style":165},[133],[41,1667,87],{"className":1668},[169],[41,1670],{"className":1671,"style":165},[133],[41,1673,1675,1678,1681,1684,1687,1690],{"className":1674},[112],[41,1676],{"className":1677,"style":117},[116],[41,1679,75],{"className":1680},[121,125],[41,1682,79],{"className":1683},[154],[41,1685,94],{"className":1686},[121],[41,1688,66],{"className":1689,"style":126},[121,125],[41,1691,84],{"className":1692},[161],[27,1694,1695,1696,729,1725,1760],{},"Distributional symmetry: ",[41,1697,1699,1713],{"className":1698},[44],[41,1700,1702],{"className":1701},[48],[50,1703,1704],{"xmlns":52},[54,1705,1706,1710],{},[57,1707,1708],{},[60,1709,268],{},[100,1711,1712],{"encoding":102},"\\sigma",[41,1714,1716],{"className":1715,"ariaHidden":108},[107],[41,1717,1719,1722],{"className":1718},[112],[41,1720],{"className":1721,"style":1327},[116],[41,1723,268],{"className":1724,"style":461},[121,125],[41,1726,1728,1744],{"className":1727},[44],[41,1729,1731],{"className":1730},[48],[50,1732,1733],{"xmlns":52},[54,1734,1735,1741],{},[57,1736,1737,1739],{},[70,1738,94],{},[60,1740,268],{},[100,1742,1743],{"encoding":102},"-\\sigma",[41,1745,1747],{"className":1746,"ariaHidden":108},[107],[41,1748,1750,1754,1757],{"className":1749},[112],[41,1751],{"className":1752,"style":1753},[116],"height:0.6667em;vertical-align:-0.0833em;",[41,1755,94],{"className":1756},[121],[41,1758,268],{"className":1759,"style":461},[121,125]," are identically distributed",[27,1762,1763],{},"Rademacher averaging: Cancellation of terms through symmetry arguments",[20,1765,1766],{},"This establishes tighter bounds by leveraging the activation function's inherent structure.",[15,1768,1770],{"id":1769},"part-c-scale-invariant-complexity-measure","Part C: Scale-Invariant Complexity Measure",[20,1772,1773,1774,1777,1778,1868,1869,1923,1924],{},"Leveraging the ",[30,1775,1776],{},"positive homogeneity"," of ReLU (",[41,1779,1781,1816],{"className":1780},[44],[41,1782,1784],{"className":1783},[48],[50,1785,1786],{"xmlns":52},[54,1787,1788,1813],{},[57,1789,1790,1792,1794,1797,1799,1801,1803,1805,1807,1809,1811],{},[60,1791,268],{},[70,1793,79],{"stretchy":78},[60,1795,1796],{},"λ",[60,1798,66],{},[70,1800,84],{"stretchy":78},[70,1802,72],{},[60,1804,1796],{},[60,1806,268],{},[70,1808,79],{"stretchy":78},[60,1810,66],{},[70,1812,84],{"stretchy":78},[100,1814,1815],{"encoding":102},"\\sigma(\\lambda z) = \\lambda \\sigma(z)",[41,1817,1819,1849],{"className":1818,"ariaHidden":108},[107],[41,1820,1822,1825,1828,1831,1834,1837,1840,1843,1846],{"className":1821},[112],[41,1823],{"className":1824,"style":117},[116],[41,1826,268],{"className":1827,"style":461},[121,125],[41,1829,79],{"className":1830},[154],[41,1832,1796],{"className":1833},[121,125],[41,1835,66],{"className":1836,"style":126},[121,125],[41,1838,84],{"className":1839},[161],[41,1841],{"className":1842,"style":134},[133],[41,1844,72],{"className":1845},[138],[41,1847],{"className":1848,"style":134},[133],[41,1850,1852,1855,1859,1862,1865],{"className":1851},[112],[41,1853],{"className":1854,"style":117},[116],[41,1856,1858],{"className":1857,"style":461},[121,125],"λσ",[41,1860,79],{"className":1861},[154],[41,1863,66],{"className":1864,"style":126},[121,125],[41,1866,84],{"className":1867},[161]," for ",[41,1870,1872,1892],{"className":1871},[44],[41,1873,1875],{"className":1874},[48],[50,1876,1877],{"xmlns":52},[54,1878,1879,1889],{},[57,1880,1881,1883,1886],{},[60,1882,1796],{},[70,1884,1885],{},">",[252,1887,1888],{},"0",[100,1890,1891],{"encoding":102},"\\lambda > 0",[41,1893,1895,1914],{"className":1894,"ariaHidden":108},[107],[41,1896,1898,1902,1905,1908,1911],{"className":1897},[112],[41,1899],{"className":1900,"style":1901},[116],"height:0.7335em;vertical-align:-0.0391em;",[41,1903,1796],{"className":1904},[121,125],[41,1906],{"className":1907,"style":134},[133],[41,1909,1885],{"className":1910},[138],[41,1912],{"className":1913,"style":134},[133],[41,1915,1917,1920],{"className":1916},[112],[41,1918],{"className":1919,"style":1518},[116],[41,1921,1888],{"className":1922},[121],"), we introduce scale-invariant parameterization: ",[41,1925,1927,2011],{"className":1926},[44],[41,1928,1930],{"className":1929},[48],[50,1931,1932],{"xmlns":52},[54,1933,1934,2008],{},[57,1935,1936,1948,1950,1970,1973,1976,1986,1988],{},[259,1937,1938,1946],{},[1939,1940,1941,1943],"mover",{"accent":108},[60,1942,263],{},[70,1944,1945],{},"^",[60,1947,248],{},[70,1949,72],{},[938,1951,1952,1958],{},[259,1953,1954,1956],{},[60,1955,263],{},[60,1957,248],{},[57,1959,1960,1962,1968],{},[60,1961,590],{"mathvariant":62},[259,1963,1964,1966],{},[60,1965,275],{},[60,1967,248],{},[60,1969,590],{"mathvariant":62},[70,1971,1972],{"separator":108},",",[133,1974],{"width":1975},"1em",[259,1977,1978,1984],{},[1939,1979,1980,1982],{"accent":108},[60,1981,275],{},[70,1983,1945],{},[60,1985,248],{},[70,1987,72],{},[938,1989,1990,1996],{},[259,1991,1992,1994],{},[60,1993,275],{},[60,1995,248],{},[57,1997,1998,2000,2006],{},[60,1999,590],{"mathvariant":62},[259,2001,2002,2004],{},[60,2003,275],{},[60,2005,248],{},[60,2007,590],{"mathvariant":62},[100,2009,2010],{"encoding":102},"\\hat{u}_j = \\frac{u_j}{\\|w_j\\|}, \\quad \\hat{w}_j = \\frac{w_j}{\\|w_j\\|}",[41,2012,2014,2103,2354],{"className":2013,"ariaHidden":108},[107],[41,2015,2017,2021,2094,2097,2100],{"className":2016},[112],[41,2018],{"className":2019,"style":2020},[116],"height:0.9805em;vertical-align:-0.2861em;",[41,2022,2024,2060],{"className":2023},[121],[41,2025,2028],{"className":2026},[121,2027],"accent",[41,2029,2031],{"className":2030},[343],[41,2032,2034],{"className":2033},[348],[41,2035,2038,2047],{"className":2036,"style":2037},[352],"height:0.6944em;",[41,2039,2041,2044],{"style":2040},"top:-3em;",[41,2042],{"className":2043,"style":1213},[360],[41,2045,263],{"className":2046},[121,125],[41,2048,2049,2052],{"style":2040},[41,2050],{"className":2051,"style":1213},[360],[41,2053,2057],{"className":2054,"style":2056},[2055],"accent-body","left:-0.2222em;",[41,2058,1945],{"className":2059},[121],[41,2061,2063],{"className":2062},[339],[41,2064,2066,2086],{"className":2065},[343,344],[41,2067,2069,2083],{"className":2068},[348],[41,2070,2072],{"className":2071,"style":433},[352],[41,2073,2074,2077],{"style":436},[41,2075],{"className":2076,"style":361},[360],[41,2078,2080],{"className":2079},[365,366,367,368],[41,2081,248],{"className":2082,"style":375},[121,125,368],[41,2084,401],{"className":2085},[400],[41,2087,2089],{"className":2088},[348],[41,2090,2092],{"className":2091,"style":455},[352],[41,2093],{},[41,2095],{"className":2096,"style":134},[133],[41,2098,72],{"className":2099},[138],[41,2101],{"className":2102,"style":134},[133],[41,2104,2106,2110,2265,2269,2273,2276,2345,2348,2351],{"className":2105},[112],[41,2107],{"className":2108,"style":2109},[116],"height:1.351em;vertical-align:-0.5423em;",[41,2111,2113,2116,2262],{"className":2112},[121],[41,2114],{"className":2115},[154,1193],[41,2117,2119],{"className":2118},[938],[41,2120,2122,2253],{"className":2121},[343,344],[41,2123,2125,2250],{"className":2124},[348],[41,2126,2129,2189,2197],{"className":2127,"style":2128},[352],"height:0.8087em;",[41,2130,2131,2134],{"style":1209},[41,2132],{"className":2133,"style":1213},[360],[41,2135,2137],{"className":2136},[365,366,367,368],[41,2138,2140,2143,2186],{"className":2139},[121,368],[41,2141,590],{"className":2142},[121,368],[41,2144,2146,2149],{"className":2145},[121,368],[41,2147,275],{"className":2148,"style":471},[121,125,368],[41,2150,2152],{"className":2151},[339],[41,2153,2155,2177],{"className":2154},[343,344],[41,2156,2158,2174],{"className":2157},[348],[41,2159,2162],{"className":2160,"style":2161},[352],"height:0.3281em;",[41,2163,2165,2168],{"style":2164},"top:-2.357em;margin-left:-0.0269em;margin-right:0.0714em;",[41,2166],{"className":2167,"style":1014},[360],[41,2169,2171],{"className":2170},[365,1018,1019,368],[41,2172,248],{"className":2173,"style":375},[121,125,368],[41,2175,401],{"className":2176},[400],[41,2178,2180],{"className":2179},[348],[41,2181,2184],{"className":2182,"style":2183},[352],"height:0.2819em;",[41,2185],{},[41,2187,590],{"className":2188},[121,368],[41,2190,2191,2194],{"style":1225},[41,2192],{"className":2193,"style":1213},[360],[41,2195],{"className":2196,"style":1233},[1232],[41,2198,2200,2203],{"style":2199},"top:-3.5073em;",[41,2201],{"className":2202,"style":1213},[360],[41,2204,2206],{"className":2205},[365,366,367,368],[41,2207,2209],{"className":2208},[121,368],[41,2210,2212,2215],{"className":2211},[121,368],[41,2213,263],{"className":2214},[121,125,368],[41,2216,2218],{"className":2217},[339],[41,2219,2221,2242],{"className":2220},[343,344],[41,2222,2224,2239],{"className":2223},[348],[41,2225,2227],{"className":2226,"style":2161},[352],[41,2228,2230,2233],{"style":2229},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[41,2231],{"className":2232,"style":1014},[360],[41,2234,2236],{"className":2235},[365,1018,1019,368],[41,2237,248],{"className":2238,"style":375},[121,125,368],[41,2240,401],{"className":2241},[400],[41,2243,2245],{"className":2244},[348],[41,2246,2248],{"className":2247,"style":2183},[352],[41,2249],{},[41,2251,401],{"className":2252},[400],[41,2254,2256],{"className":2255},[348],[41,2257,2260],{"className":2258,"style":2259},[352],"height:0.5423em;",[41,2261],{},[41,2263],{"className":2264},[161,1193],[41,2266,1972],{"className":2267},[2268],"mpunct",[41,2270],{"className":2271,"style":2272},[133],"margin-right:1em;",[41,2274],{"className":2275,"style":414},[133],[41,2277,2279,2311],{"className":2278},[121],[41,2280,2282],{"className":2281},[121,2027],[41,2283,2285],{"className":2284},[343],[41,2286,2288],{"className":2287},[348],[41,2289,2291,2299],{"className":2290,"style":2037},[352],[41,2292,2293,2296],{"style":2040},[41,2294],{"className":2295,"style":1213},[360],[41,2297,275],{"className":2298,"style":471},[121,125],[41,2300,2301,2304],{"style":2040},[41,2302],{"className":2303,"style":1213},[360],[41,2305,2308],{"className":2306,"style":2307},[2055],"left:-0.1667em;",[41,2309,1945],{"className":2310},[121],[41,2312,2314],{"className":2313},[339],[41,2315,2317,2337],{"className":2316},[343,344],[41,2318,2320,2334],{"className":2319},[348],[41,2321,2323],{"className":2322,"style":433},[352],[41,2324,2325,2328],{"style":645},[41,2326],{"className":2327,"style":361},[360],[41,2329,2331],{"className":2330},[365,366,367,368],[41,2332,248],{"className":2333,"style":375},[121,125,368],[41,2335,401],{"className":2336},[400],[41,2338,2340],{"className":2339},[348],[41,2341,2343],{"className":2342,"style":455},[352],[41,2344],{},[41,2346],{"className":2347,"style":134},[133],[41,2349,72],{"className":2350},[138],[41,2352],{"className":2353,"style":134},[133],[41,2355,2357,2360],{"className":2356},[112],[41,2358],{"className":2359,"style":2109},[116],[41,2361,2363,2366,2505],{"className":2362},[121],[41,2364],{"className":2365},[154,1193],[41,2367,2369],{"className":2368},[938],[41,2370,2372,2497],{"className":2371},[343,344],[41,2373,2375,2494],{"className":2374},[348],[41,2376,2378,2435,2443],{"className":2377,"style":2128},[352],[41,2379,2380,2383],{"style":1209},[41,2381],{"className":2382,"style":1213},[360],[41,2384,2386],{"className":2385},[365,366,367,368],[41,2387,2389,2392,2432],{"className":2388},[121,368],[41,2390,590],{"className":2391},[121,368],[41,2393,2395,2398],{"className":2394},[121,368],[41,2396,275],{"className":2397,"style":471},[121,125,368],[41,2399,2401],{"className":2400},[339],[41,2402,2404,2424],{"className":2403},[343,344],[41,2405,2407,2421],{"className":2406},[348],[41,2408,2410],{"className":2409,"style":2161},[352],[41,2411,2412,2415],{"style":2164},[41,2413],{"className":2414,"style":1014},[360],[41,2416,2418],{"className":2417},[365,1018,1019,368],[41,2419,248],{"className":2420,"style":375},[121,125,368],[41,2422,401],{"className":2423},[400],[41,2425,2427],{"className":2426},[348],[41,2428,2430],{"className":2429,"style":2183},[352],[41,2431],{},[41,2433,590],{"className":2434},[121,368],[41,2436,2437,2440],{"style":1225},[41,2438],{"className":2439,"style":1213},[360],[41,2441],{"className":2442,"style":1233},[1232],[41,2444,2445,2448],{"style":2199},[41,2446],{"className":2447,"style":1213},[360],[41,2449,2451],{"className":2450},[365,366,367,368],[41,2452,2454],{"className":2453},[121,368],[41,2455,2457,2460],{"className":2456},[121,368],[41,2458,275],{"className":2459,"style":471},[121,125,368],[41,2461,2463],{"className":2462},[339],[41,2464,2466,2486],{"className":2465},[343,344],[41,2467,2469,2483],{"className":2468},[348],[41,2470,2472],{"className":2471,"style":2161},[352],[41,2473,2474,2477],{"style":2164},[41,2475],{"className":2476,"style":1014},[360],[41,2478,2480],{"className":2479},[365,1018,1019,368],[41,2481,248],{"className":2482,"style":375},[121,125,368],[41,2484,401],{"className":2485},[400],[41,2487,2489],{"className":2488},[348],[41,2490,2492],{"className":2491,"style":2183},[352],[41,2493],{},[41,2495,401],{"className":2496},[400],[41,2498,2500],{"className":2499},[348],[41,2501,2503],{"className":2502,"style":2259},[352],[41,2504],{},[41,2506],{"className":2507},[161,1193],[20,2509,2510,2511],{},"This transformation yields a width-independent bound: ",[41,2512,2514,2557],{"className":2513},[44],[41,2515,2517],{"className":2516},[48],[50,2518,2519],{"xmlns":52},[54,2520,2521,2554],{},[57,2522,2523,2525,2527,2529,2531,2533,2535,2537,2539],{},[60,2524,898],{"mathvariant":897},[70,2526,79],{"stretchy":78},[60,2528,911],{"mathvariant":897},[70,2530,84],{"stretchy":78},[70,2532,601],{},[60,2534,933],{},[70,2536,548],{},[60,2538,606],{},[935,2540,2541],{},[938,2542,2543,2552],{},[57,2544,2545,2548,2550],{},[60,2546,2547],{},"log",[70,2549,1419],{},[60,2551,257],{},[60,2553,906],{},[100,2555,2556],{"encoding":102},"\\mathcal{R}(\\mathcal{H}) \\leq C \\cdot B \\sqrt{\\frac{\\log m}{n}}",[41,2558,2560,2587,2606],{"className":2559,"ariaHidden":108},[107],[41,2561,2563,2566,2569,2572,2575,2578,2581,2584],{"className":2562},[112],[41,2564],{"className":2565,"style":117},[116],[41,2567,898],{"className":2568},[121,963],[41,2570,79],{"className":2571},[154],[41,2573,911],{"className":2574,"style":1053},[121,963],[41,2576,84],{"className":2577},[161],[41,2579],{"className":2580,"style":134},[133],[41,2582,601],{"className":2583},[138],[41,2585],{"className":2586,"style":134},[133],[41,2588,2590,2594,2597,2600,2603],{"className":2589},[112],[41,2591],{"className":2592,"style":2593},[116],"height:0.6833em;",[41,2595,933],{"className":2596,"style":1159},[121,125],[41,2598],{"className":2599,"style":165},[133],[41,2601,548],{"className":2602},[169],[41,2604],{"className":2605,"style":165},[133],[41,2607,2609,2613,2616],{"className":2608},[112],[41,2610],{"className":2611,"style":2612},[116],"height:1.84em;vertical-align:-0.5614em;",[41,2614,606],{"className":2615,"style":691},[121,125],[41,2617,2619],{"className":2618},[121,1163],[41,2620,2622,2750],{"className":2621},[343,344],[41,2623,2625,2747],{"className":2624},[348],[41,2626,2629,2730],{"className":2627,"style":2628},[352],"height:1.2786em;",[41,2630,2633,2637],{"className":2631,"style":2632},[1177],"top:-3.8em;",[41,2634],{"className":2635,"style":2636},[360],"height:3.8em;",[41,2638,2640],{"className":2639,"style":1186},[121],[41,2641,2643,2646,2727],{"className":2642},[121],[41,2644],{"className":2645},[154,1193],[41,2647,2649],{"className":2648},[938],[41,2650,2652,2719],{"className":2651},[343,344],[41,2653,2655,2716],{"className":2654},[348],[41,2656,2659,2673,2681],{"className":2657,"style":2658},[352],"height:0.9322em;",[41,2660,2661,2664],{"style":1209},[41,2662],{"className":2663,"style":1213},[360],[41,2665,2667],{"className":2666},[365,366,367,368],[41,2668,2670],{"className":2669},[121,368],[41,2671,906],{"className":2672},[121,125,368],[41,2674,2675,2678],{"style":1225},[41,2676],{"className":2677,"style":1213},[360],[41,2679],{"className":2680,"style":1233},[1232],[41,2682,2684,2687],{"style":2683},"top:-3.4461em;",[41,2685],{"className":2686,"style":1213},[360],[41,2688,2690],{"className":2689},[365,366,367,368],[41,2691,2693,2709,2713],{"className":2692},[121,368],[41,2694,2696,2700,2704],{"className":2695},[329,368],[41,2697,2699],{"className":2698},[368],"l",[41,2701,2703],{"className":2702},[368],"o",[41,2705,2708],{"className":2706,"style":2707},[368],"margin-right:0.0139em;","g",[41,2710],{"className":2711,"style":2712},[133,368],"margin-right:0.1952em;",[41,2714,257],{"className":2715},[121,125,368],[41,2717,401],{"className":2718},[400],[41,2720,2722],{"className":2721},[348],[41,2723,2725],{"className":2724,"style":1258},[352],[41,2726],{},[41,2728],{"className":2729},[161,1193],[41,2731,2733,2736],{"style":2732},"top:-3.2386em;",[41,2734],{"className":2735,"style":2636},[360],[41,2737,2740],{"className":2738,"style":2739},[1273],"min-width:1.02em;height:1.88em;",[1276,2741,2744],{"xmlns":1278,"width":1279,"height":2742,"viewBox":2743,"preserveAspectRatio":1282},"1.88em","0 0 400000 1944",[1284,2745],{"d":2746},"M983 90\nl0 -0\nc4,-6.7,10,-10,18,-10 H400000v40\nH1013.1s-83.4,268,-264.1,840c-180.7,572,-277,876.3,-289,913c-4.7,4.7,-12.7,7,-24,7\ns-12,0,-12,0c-1.3,-3.3,-3.7,-11.7,-7,-25c-35.3,-125.3,-106.7,-373.3,-214,-744\nc-10,12,-21,25,-33,39s-32,39,-32,39c-6,-5.3,-15,-14,-27,-26s25,-30,25,-30\nc26.7,-32.7,52,-63,76,-91s52,-60,52,-60s208,722,208,722\nc56,-175.3,126.3,-397.3,211,-666c84.7,-268.7,153.8,-488.2,207.5,-658.5\nc53.7,-170.3,84.5,-266.8,92.5,-289.5z\nM1001 80h400000v40h-400000z",[41,2748,401],{"className":2749},[400],[41,2751,2753],{"className":2752},[348],[41,2754,2757],{"className":2755,"style":2756},[352],"height:0.5614em;",[41,2758],{},[20,2760,2761,2762,2790],{},"where the complexity no longer scales linearly with ",[41,2763,2765,2778],{"className":2764},[44],[41,2766,2768],{"className":2767},[48],[50,2769,2770],{"xmlns":52},[54,2771,2772,2776],{},[57,2773,2774],{},[60,2775,257],{},[100,2777,257],{"encoding":102},[41,2779,2781],{"className":2780,"ariaHidden":108},[107],[41,2782,2784,2787],{"className":2783},[112],[41,2785],{"className":2786,"style":1327},[116],[41,2788,257],{"className":2789},[121,125],", making it meaningful for overparameterized networks.",[15,2792,2794],{"id":2793},"course-academic-context","Course & Academic Context",[24,2796,2797,2803,2809,2815,2821],{},[27,2798,2799,2802],{},[30,2800,2801],{},"Course",": Learning Theory: Two-Layer Neural Networks (Exam)",[27,2804,2805,2808],{},[30,2806,2807],{},"Institution",": Université Paris-Dauphine -- PSL, Department of MIDO",[27,2810,2811,2814],{},[30,2812,2813],{},"Supervisor",": Katia Meziani",[27,2816,2817,2820],{},[30,2818,2819],{},"Program",": Master 2 ISF (Initial Track)",[27,2822,2823,2826],{},[30,2824,2825],{},"Academic Year",": 2025\u002F2026",[15,2828,2830],{"id":2829},"detailed-report","Detailed Report",[2832,2833],"iframe",{"src":2834,"width":2835,"height":2836},"\u002Fprojects\u002Flt-two-layer-neural-networks.pdf","100%","1000px",{"title":2838,"searchDepth":2839,"depth":2839,"links":2840},"",2,[2841,2842,2843,2844,2845,2846,2847],{"id":17,"depth":2839,"text":18},{"id":209,"depth":2839,"text":210},{"id":876,"depth":2839,"text":877},{"id":1388,"depth":2839,"text":1389},{"id":1769,"depth":2839,"text":1770},{"id":2793,"depth":2839,"text":2794},{"id":2829,"depth":2839,"text":2830},"2026-04-12",4,"Derivation of tight generalization bounds for two-layer ReLU networks using scale-invariance and symmetrization inequalities.","lt-two-layers-neural-networks","Completed","projects\u002Flt-two-layers-neural-networks",[2855,2856,2857,2858],"Neural Networks","Generalization Bounds","ReLU Activation","Rademacher Complexity","Research Project","X6UvozyXN16DAhoamf-82JXbmRyj4FXZmT0xK8q_2ds",1777982163834]