diff --git a/Kconfig b/Kconfig
index 6957097..78afe72 100644
--- a/Kconfig
+++ b/Kconfig
@@ -361,6 +361,16 @@ endif # SPL
 
 endif # FIT
 
+config IMAGE_FORMAT_LEGACY
+	bool "Enable support for the legacy image format"
+	default y if !FIT_SIGNATURE
+	help
+	  This option enables the legacy image format. It is enabled by
+	  default for backward compatibility, unless FIT_SIGNATURE is
+	  set where it is disabled so that unsigned images cannot be
+	  loaded. If a board needs the legacy image format support in this
+	  case, enable it here.
+
 config OF_BOARD_SETUP
 	bool "Set up board-specific details in device tree before boot"
 	depends on OF_LIBFDT
diff --git a/README b/README
index 6f98e09..a15a3dd 100644
--- a/README
+++ b/README
@@ -2413,19 +2413,6 @@ FIT uImage format:
  -150	common/cmd_nand.c	Incorrect FIT image format
   151	common/cmd_nand.c	FIT image format OK
 
-- legacy image format:
-		CONFIG_IMAGE_FORMAT_LEGACY
-		enables the legacy image format support in U-Boot.
-
-		Default:
-		enabled if CONFIG_FIT_SIGNATURE is not defined.
-
-		CONFIG_DISABLE_IMAGE_LEGACY
-		disable the legacy image format
-
-		This define is introduced, as the legacy image format is
-		enabled per default for backward compatibility.
-
 - Standalone program support:
 		CONFIG_STANDALONE_LOAD_ADDR
 
diff --git a/configs/ids8313_defconfig b/configs/ids8313_defconfig
index b662111..695a1ed 100644
--- a/configs/ids8313_defconfig
+++ b/configs/ids8313_defconfig
@@ -4,6 +4,7 @@ CONFIG_MPC83xx=y
 CONFIG_TARGET_IDS8313=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_OF_BOARD_SETUP=y
 CONFIG_OF_STDOUT_VIA_ALIAS=y
 CONFIG_BOOTDELAY=1
diff --git a/configs/imx6dl_icore_nand_defconfig b/configs/imx6dl_icore_nand_defconfig
index 74a534c..c34e3ac 100644
--- a/configs/imx6dl_icore_nand_defconfig
+++ b/configs/imx6dl_icore_nand_defconfig
@@ -13,6 +13,7 @@ CONFIG_DEFAULT_DEVICE_TREE="imx6dl-icore"
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SPL_DMA_SUPPORT=y
diff --git a/configs/imx6q_icore_nand_defconfig b/configs/imx6q_icore_nand_defconfig
index 0caba80..f8477a9 100644
--- a/configs/imx6q_icore_nand_defconfig
+++ b/configs/imx6q_icore_nand_defconfig
@@ -13,6 +13,7 @@ CONFIG_DEFAULT_DEVICE_TREE="imx6q-icore"
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6qdl_icore_mipi_defconfig b/configs/imx6qdl_icore_mipi_defconfig
index 4e1dad6..17fff72 100644
--- a/configs/imx6qdl_icore_mipi_defconfig
+++ b/configs/imx6qdl_icore_mipi_defconfig
@@ -17,6 +17,7 @@ CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6qdl_icore_mmc_defconfig b/configs/imx6qdl_icore_mmc_defconfig
index 308e11c..948894e 100644
--- a/configs/imx6qdl_icore_mmc_defconfig
+++ b/configs/imx6qdl_icore_mmc_defconfig
@@ -17,6 +17,7 @@ CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6qdl_icore_nand_defconfig b/configs/imx6qdl_icore_nand_defconfig
index 0caba80..f8477a9 100644
--- a/configs/imx6qdl_icore_nand_defconfig
+++ b/configs/imx6qdl_icore_nand_defconfig
@@ -13,6 +13,7 @@ CONFIG_DEFAULT_DEVICE_TREE="imx6q-icore"
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6qdl_icore_rqs_defconfig b/configs/imx6qdl_icore_rqs_defconfig
index fb7f260..2e49fe3 100644
--- a/configs/imx6qdl_icore_rqs_defconfig
+++ b/configs/imx6qdl_icore_rqs_defconfig
@@ -16,6 +16,7 @@ CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6ul_geam_mmc_defconfig b/configs/imx6ul_geam_mmc_defconfig
index b4aa1b2..6530667 100644
--- a/configs/imx6ul_geam_mmc_defconfig
+++ b/configs/imx6ul_geam_mmc_defconfig
@@ -15,6 +15,7 @@ CONFIG_DEFAULT_DEVICE_TREE="imx6ul-geam-kit"
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6ul_geam_nand_defconfig b/configs/imx6ul_geam_nand_defconfig
index f836c18..243ab39 100644
--- a/configs/imx6ul_geam_nand_defconfig
+++ b/configs/imx6ul_geam_nand_defconfig
@@ -13,6 +13,7 @@ CONFIG_DEFAULT_DEVICE_TREE="imx6ul-geam-kit"
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6ul_isiot_emmc_defconfig b/configs/imx6ul_isiot_emmc_defconfig
index 2a5dbef..1fa41f6 100644
--- a/configs/imx6ul_isiot_emmc_defconfig
+++ b/configs/imx6ul_isiot_emmc_defconfig
@@ -15,6 +15,7 @@ CONFIG_DEFAULT_DEVICE_TREE="imx6ul-isiot-emmc"
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/imx6ul_isiot_nand_defconfig b/configs/imx6ul_isiot_nand_defconfig
index 2c84424..d738c0b 100644
--- a/configs/imx6ul_isiot_nand_defconfig
+++ b/configs/imx6ul_isiot_nand_defconfig
@@ -13,6 +13,7 @@ CONFIG_DEFAULT_DEVICE_TREE="imx6ul-isiot-nand"
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_SYS_EXTRA_OPTIONS="IMX_CONFIG=arch/arm/mach-imx/spl_sd.cfg"
 CONFIG_BOOTDELAY=3
 CONFIG_SUPPORT_RAW_INITRD=y
diff --git a/configs/syzygy_hub_defconfig b/configs/syzygy_hub_defconfig
index 005cb9c..aef21a8 100644
--- a/configs/syzygy_hub_defconfig
+++ b/configs/syzygy_hub_defconfig
@@ -11,6 +11,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_cc108_defconfig b/configs/zynq_cc108_defconfig
index 170cfbd..f161cf8 100644
--- a/configs/zynq_cc108_defconfig
+++ b/configs/zynq_cc108_defconfig
@@ -9,6 +9,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_microzed_defconfig b/configs/zynq_microzed_defconfig
index 967c037..34939fa 100644
--- a/configs/zynq_microzed_defconfig
+++ b/configs/zynq_microzed_defconfig
@@ -8,6 +8,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_z_turn_defconfig b/configs/zynq_z_turn_defconfig
index 5b13edb..c2c478d 100644
--- a/configs/zynq_z_turn_defconfig
+++ b/configs/zynq_z_turn_defconfig
@@ -9,6 +9,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zc702_defconfig b/configs/zynq_zc702_defconfig
index 2c9210f..c23dd8f 100644
--- a/configs/zynq_zc702_defconfig
+++ b/configs/zynq_zc702_defconfig
@@ -11,6 +11,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zc706_defconfig b/configs/zynq_zc706_defconfig
index dd1b1af..00042c1 100644
--- a/configs/zynq_zc706_defconfig
+++ b/configs/zynq_zc706_defconfig
@@ -11,6 +11,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zc770_xm010_defconfig b/configs/zynq_zc770_xm010_defconfig
index 214d7eb..7713e5a 100644
--- a/configs/zynq_zc770_xm010_defconfig
+++ b/configs/zynq_zc770_xm010_defconfig
@@ -10,6 +10,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zc770_xm011_defconfig b/configs/zynq_zc770_xm011_defconfig
index 8db7b99..3c394bf 100644
--- a/configs/zynq_zc770_xm011_defconfig
+++ b/configs/zynq_zc770_xm011_defconfig
@@ -11,6 +11,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zc770_xm011_x16_defconfig b/configs/zynq_zc770_xm011_x16_defconfig
index b53ca3d..4bc49b3 100644
--- a/configs/zynq_zc770_xm011_x16_defconfig
+++ b/configs/zynq_zc770_xm011_x16_defconfig
@@ -11,6 +11,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zc770_xm012_defconfig b/configs/zynq_zc770_xm012_defconfig
index b3956ee..0a78830 100644
--- a/configs/zynq_zc770_xm012_defconfig
+++ b/configs/zynq_zc770_xm012_defconfig
@@ -10,6 +10,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zc770_xm013_defconfig b/configs/zynq_zc770_xm013_defconfig
index 76f881a..6659218 100644
--- a/configs/zynq_zc770_xm013_defconfig
+++ b/configs/zynq_zc770_xm013_defconfig
@@ -10,6 +10,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zed_defconfig b/configs/zynq_zed_defconfig
index 769c531..13898b3 100644
--- a/configs/zynq_zed_defconfig
+++ b/configs/zynq_zed_defconfig
@@ -9,6 +9,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/configs/zynq_zybo_defconfig b/configs/zynq_zybo_defconfig
index fbe22c5..771d8ca 100644
--- a/configs/zynq_zybo_defconfig
+++ b/configs/zynq_zybo_defconfig
@@ -10,6 +10,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
+CONFIG_IMAGE_FORMAT_LEGACY=y
 CONFIG_BOOTCOMMAND="run $modeboot || run distro_bootcmd"
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_STACK_R=y
diff --git a/include/config_fallbacks.h b/include/config_fallbacks.h
index 23a4cfb..c18f19a 100644
--- a/include/config_fallbacks.h
+++ b/include/config_fallbacks.h
@@ -46,14 +46,6 @@
 #define CONFIG_SYS_MAXARGS	16
 #endif
 
-#ifndef CONFIG_FIT_SIGNATURE
-#define CONFIG_IMAGE_FORMAT_LEGACY
-#endif
-
-#ifdef CONFIG_DISABLE_IMAGE_LEGACY
-#undef CONFIG_IMAGE_FORMAT_LEGACY
-#endif
-
 #ifdef CONFIG_DM_I2C
 # ifdef CONFIG_SYS_I2C
 #  error "Cannot define CONFIG_SYS_I2C when CONFIG_DM_I2C is used"
diff --git a/include/configs/ids8313.h b/include/configs/ids8313.h
index a10dc12..8357895 100644
--- a/include/configs/ids8313.h
+++ b/include/configs/ids8313.h
@@ -506,6 +506,4 @@
 /* UBI Support */
 #define CONFIG_MTD_PARTITIONS
 
-#define CONFIG_IMAGE_FORMAT_LEGACY
-
 #endif	/* __CONFIG_H */
diff --git a/include/configs/imx6-engicam.h b/include/configs/imx6-engicam.h
index 246a636..86604d6 100644
--- a/include/configs/imx6-engicam.h
+++ b/include/configs/imx6-engicam.h
@@ -130,11 +130,6 @@
 #define CONFIG_SYS_INIT_SP_ADDR		(CONFIG_SYS_INIT_RAM_ADDR + \
 					CONFIG_SYS_INIT_SP_OFFSET)
 
-/* FIT */
-#ifdef CONFIG_FIT
-# define CONFIG_IMAGE_FORMAT_LEGACY
-#endif
-
 /* UART */
 #ifdef CONFIG_MXC_UART
 # ifdef CONFIG_MX6UL
diff --git a/include/configs/wb50n.h b/include/configs/wb50n.h
index 028e3ff..093cc32 100644
--- a/include/configs/wb50n.h
+++ b/include/configs/wb50n.h
@@ -23,8 +23,6 @@
 #define CONFIG_SKIP_LOWLEVEL_INIT
 #endif
 
-#define CONFIG_IMAGE_FORMAT_LEGACY
-
 /* general purpose I/O */
 #define CONFIG_AT91_GPIO
 
diff --git a/include/configs/zynq-common.h b/include/configs/zynq-common.h
index ae82a7a..9872f0a 100644
--- a/include/configs/zynq-common.h
+++ b/include/configs/zynq-common.h
@@ -259,9 +259,6 @@
 					GENERATED_GBL_DATA_SIZE)
 
 
-/* FIT support */
-#define CONFIG_IMAGE_FORMAT_LEGACY /* enable also legacy image format */
-
 /* Extend size of kernel image for uncompression */
 #define CONFIG_SYS_BOOTM_LEN	(60 * 1024 * 1024)
 
diff --git a/scripts/config_whitelist.txt b/scripts/config_whitelist.txt
index 193c7de..517805a 100644
--- a/scripts/config_whitelist.txt
+++ b/scripts/config_whitelist.txt
@@ -373,7 +373,6 @@ CONFIG_DIALOG_POWER
 CONFIG_DIMM_SLOTS_PER_CTLR
 CONFIG_DIRECT_NOR_BOOT
 CONFIG_DISABLE_CONSOLE
-CONFIG_DISABLE_IMAGE_LEGACY
 CONFIG_DISCONTIGMEM
 CONFIG_DISCOVER_PHY
 CONFIG_DISPLAY_AER_xxxx
@@ -974,7 +973,6 @@ CONFIG_IDE_SWAP_IO
 CONFIG_IDT8T49N222A
 CONFIG_ID_EEPROM
 CONFIG_IMA
-CONFIG_IMAGE_FORMAT_LEGACY
 CONFIG_IMX
 CONFIG_IMX6_PWM_PER_CLK
 CONFIG_IMX_HDMI
