diff --git a/Makefile b/Makefile
index c4d0c3002e..ce2ca00730 100644
--- a/Makefile
+++ b/Makefile
@@ -712,8 +712,8 @@ libs-y += drivers/usb/dwc3/
 libs-y += drivers/usb/common/
 libs-y += drivers/usb/emul/
 libs-y += drivers/usb/eth/
-libs-y += drivers/usb/gadget/
-libs-y += drivers/usb/gadget/udc/
+libs-$(CONFIG_USB_GADGET) += drivers/usb/gadget/
+libs-$(CONFIG_USB_GADGET) += drivers/usb/gadget/udc/
 libs-y += drivers/usb/host/
 libs-y += drivers/usb/musb/
 libs-y += drivers/usb/musb-new/
diff --git a/configs/cm_t3517_defconfig b/configs/cm_t3517_defconfig
index 0901fea638..c204a29081 100644
--- a/configs/cm_t3517_defconfig
+++ b/configs/cm_t3517_defconfig
@@ -54,6 +54,7 @@ CONFIG_USB=y
 CONFIG_USB_MUSB_HOST=y
 CONFIG_USB_MUSB_AM35X=y
 CONFIG_USB_STORAGE=y
+CONFIG_USB_GADGET=y
 CONFIG_VIDEO_OMAP3=y
 CONFIG_LCD=y
 CONFIG_OF_LIBFDT=y
diff --git a/configs/cm_t35_defconfig b/configs/cm_t35_defconfig
index f1fe2d058d..a27c502489 100644
--- a/configs/cm_t35_defconfig
+++ b/configs/cm_t35_defconfig
@@ -56,6 +56,7 @@ CONFIG_USB_MUSB_UDC=y
 CONFIG_USB_OMAP3=y
 CONFIG_TWL4030_USB=y
 CONFIG_USB_STORAGE=y
+CONFIG_USB_GADGET=y
 CONFIG_VIDEO_OMAP3=y
 CONFIG_LCD=y
 CONFIG_OF_LIBFDT=y
diff --git a/configs/duovero_defconfig b/configs/duovero_defconfig
index 8ea626d013..782a9dc291 100644
--- a/configs/duovero_defconfig
+++ b/configs/duovero_defconfig
@@ -36,5 +36,6 @@ CONFIG_USB=y
 CONFIG_USB_EHCI_HCD=y
 CONFIG_USB_MUSB_UDC=y
 CONFIG_USB_OMAP3=y
+CONFIG_USB_GADGET=y
 CONFIG_FAT_WRITE=y
 CONFIG_OF_LIBFDT=y
diff --git a/configs/igep0032_defconfig b/configs/igep0032_defconfig
index 383648789c..20d2cf5820 100644
--- a/configs/igep0032_defconfig
+++ b/configs/igep0032_defconfig
@@ -45,6 +45,7 @@ CONFIG_USB=y
 CONFIG_USB_MUSB_UDC=y
 CONFIG_USB_OMAP3=y
 CONFIG_TWL4030_USB=y
+CONFIG_USB_GADGET=y
 CONFIG_FAT_WRITE=y
 CONFIG_UBIFS_SILENCE_MSG=y
 CONFIG_BCH=y
diff --git a/configs/igep00x0_defconfig b/configs/igep00x0_defconfig
index f2989e34e1..330350c32a 100644
--- a/configs/igep00x0_defconfig
+++ b/configs/igep00x0_defconfig
@@ -46,6 +46,7 @@ CONFIG_USB=y
 CONFIG_USB_MUSB_UDC=y
 CONFIG_USB_OMAP3=y
 CONFIG_TWL4030_USB=y
+CONFIG_USB_GADGET=y
 CONFIG_FAT_WRITE=y
 CONFIG_UBIFS_SILENCE_MSG=y
 CONFIG_BCH=y
diff --git a/configs/omap3_zoom1_defconfig b/configs/omap3_zoom1_defconfig
index 325c0020ca..489e6a8ee0 100644
--- a/configs/omap3_zoom1_defconfig
+++ b/configs/omap3_zoom1_defconfig
@@ -35,5 +35,6 @@ CONFIG_USB=y
 CONFIG_USB_MUSB_UDC=y
 CONFIG_USB_OMAP3=y
 CONFIG_TWL4030_USB=y
+CONFIG_USB_GADGET=y
 CONFIG_FAT_WRITE=y
 CONFIG_OF_LIBFDT=y
diff --git a/configs/omap4_panda_defconfig b/configs/omap4_panda_defconfig
index b66659f123..7ee9eea06a 100644
--- a/configs/omap4_panda_defconfig
+++ b/configs/omap4_panda_defconfig
@@ -36,6 +36,7 @@ CONFIG_USB=y
 CONFIG_USB_EHCI_HCD=y
 CONFIG_USB_MUSB_UDC=y
 CONFIG_USB_OMAP3=y
+CONFIG_USB_GADGET=y
 CONFIG_USB_HOST_ETHER=y
 CONFIG_USB_ETHER_SMSC95XX=y
 CONFIG_OF_LIBFDT=y
diff --git a/configs/omap4_sdp4430_defconfig b/configs/omap4_sdp4430_defconfig
index 6b2ea202e5..0e01c8edcb 100644
--- a/configs/omap4_sdp4430_defconfig
+++ b/configs/omap4_sdp4430_defconfig
@@ -33,6 +33,7 @@ CONFIG_OMAP3_SPI=y
 CONFIG_USB=y
 CONFIG_USB_MUSB_UDC=y
 CONFIG_USB_OMAP3=y
+CONFIG_USB_GADGET=y
 CONFIG_FAT_WRITE=y
 # CONFIG_REGEX is not set
 CONFIG_OF_LIBFDT=y
diff --git a/configs/spear300_usbtty_defconfig b/configs/spear300_usbtty_defconfig
index bc0c3ec091..cb115fecb7 100644
--- a/configs/spear300_usbtty_defconfig
+++ b/configs/spear300_usbtty_defconfig
@@ -27,3 +27,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear300_usbtty_nand_defconfig b/configs/spear300_usbtty_nand_defconfig
index 9b90cd8638..4064ba36e9 100644
--- a/configs/spear300_usbtty_nand_defconfig
+++ b/configs/spear300_usbtty_nand_defconfig
@@ -28,3 +28,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear310_usbtty_defconfig b/configs/spear310_usbtty_defconfig
index 7786e66998..9373df83f7 100644
--- a/configs/spear310_usbtty_defconfig
+++ b/configs/spear310_usbtty_defconfig
@@ -27,3 +27,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear310_usbtty_nand_defconfig b/configs/spear310_usbtty_nand_defconfig
index 1d41522b2a..a4f09a2735 100644
--- a/configs/spear310_usbtty_nand_defconfig
+++ b/configs/spear310_usbtty_nand_defconfig
@@ -28,3 +28,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear310_usbtty_pnor_defconfig b/configs/spear310_usbtty_pnor_defconfig
index b5214b5d82..8263454de6 100644
--- a/configs/spear310_usbtty_pnor_defconfig
+++ b/configs/spear310_usbtty_pnor_defconfig
@@ -30,3 +30,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear320_usbtty_defconfig b/configs/spear320_usbtty_defconfig
index 3c10f3261e..80acff0cce 100644
--- a/configs/spear320_usbtty_defconfig
+++ b/configs/spear320_usbtty_defconfig
@@ -27,3 +27,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear320_usbtty_nand_defconfig b/configs/spear320_usbtty_nand_defconfig
index 8fb84bcc60..00534b1c24 100644
--- a/configs/spear320_usbtty_nand_defconfig
+++ b/configs/spear320_usbtty_nand_defconfig
@@ -28,3 +28,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear320_usbtty_pnor_defconfig b/configs/spear320_usbtty_pnor_defconfig
index 2cf39b450c..916321c60c 100644
--- a/configs/spear320_usbtty_pnor_defconfig
+++ b/configs/spear320_usbtty_pnor_defconfig
@@ -30,3 +30,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear600_usbtty_defconfig b/configs/spear600_usbtty_defconfig
index 5c7b0a8ab4..d0bdc7587d 100644
--- a/configs/spear600_usbtty_defconfig
+++ b/configs/spear600_usbtty_defconfig
@@ -27,3 +27,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
diff --git a/configs/spear600_usbtty_nand_defconfig b/configs/spear600_usbtty_nand_defconfig
index 078e934bba..ccbfec9298 100644
--- a/configs/spear600_usbtty_nand_defconfig
+++ b/configs/spear600_usbtty_nand_defconfig
@@ -28,3 +28,5 @@ CONFIG_PHY_GIGE=y
 CONFIG_ETH_DESIGNWARE=y
 CONFIG_MII=y
 CONFIG_CONS_INDEX=0
+CONFIG_USB=y
+CONFIG_USB_GADGET=y
