Sitecore Data Exchange Framework-Dealing with Complex fields

Data Exchange Framework is very helpful, powerful and flexible module.

In this blog post, I will show you how to import data from JSON file to complex Sitecore field using the Sitecore provider which is available in the official Sitecore download page Data Exchange Framework.

I will use the simplest and fastest way to import Complex Sitecore Fields. like date field, image field, drop link field ...etc, Which is "transform values to its Sitecore row values"

First, you need is to install data exchange framework on your Sitecore instance.

Then install Sitecore provider which is available on the same download page of data exchange framework.

Now we are ready to implement the custom JSON provider, as shown briefly below:

SC-Endpoint (JSON provider Endpoint):

It contains three fields :

1. Path - the Physical file path.
2. ItemNameSourceProperty - I use this field to name Sitecore item, which is taken from the JSON object.
3. CastTypeFullName - To convert JSON to c# Type.

VS-Plugin:

We need to read these values and store them to use them later, to store these values you have to create Plugin as below:

public class JsonFileSettings : IPlugin
    {
        public JsonFileSettings()
        {
        }
        public string Path { get; set; }
        public string ItemNameSourceProperty { get; set; }
        public string CastTypeFullName { get; set; }
    }

After that we need to implement the converter to read the values from Sitecore as below:

VS-Endpoint Converter:

public class JsonFileEndpointConverter : BaseEndpointConverter
    {
        public JsonFileEndpointConverter(IItemModelRepository repository) : base(repository)
        {
            this.SupportedTemplateIds.Add(Templates.JsonFileEndpoint.ID);
        }
        protected override void AddPlugins(ItemModel source, Endpoint endpoint)
        {
            var settings = new JsonFileSettings();
            //the file physical path
            settings.Path = base.GetStringValue(source, Templates.JsonFileEndpoint.Fields.Path);
            //This is the property will used from sitecore item naming
            settings.ItemNameSourceProperty = base.GetStringValue(source, Templates.JsonFileEndpoint.Fields.ItemNameSourceProperty);
            //The model type to parse the Json to this type
            settings.CastTypeFullName = base.GetStringValue(source, Templates.JsonFileEndpoint.Fields.CastTypeFullName);

            endpoint.Plugins.Add(settings);
        }
    }

Now, to read the values form the JSON file we need to implement the JsonValueAccessor

SC-Value Accessor:

Make the accessor inherit from the Property Value Accessor because the JSON will be converted to C# object and we will access the values by property name.

VS-Value Accessor Converter:

public class JsonValueAccessorConverter : PropertyValueAccessorConverter
    {
        public JsonValueAccessorConverter(IItemModelRepository repository) : base(repository)
        {
            this.SupportedTemplateIds.Add(Templates.JsonValueAccessor.ID);
        }
        public override IValueAccessor Convert(ItemModel source)
        {
            var accessor = base.Convert(source);
            if (accessor == null)
            {
                return null;
            }
            var propertyName = base.GetStringValue(source, Templates.JsonValueAccessor.Fields.PropertyName);
            if (string.IsNullOrWhiteSpace(propertyName))
            {
                return null;
            }

            if (accessor.ValueReader == null)
            {
                accessor.ValueReader = new PropertyValueReader(propertyName);
            }
            if (accessor.ValueWriter == null)
            {
                accessor.ValueWriter = new PropertyValueWriter(propertyName);
            }
            return accessor;
        }
    }

SC-Pipline Step:

VS-Pipline Step Converter:

public class ReadJsonFileStepConverter : BasePipelineStepConverter
    {
        public ReadJsonFileStepConverter(IItemModelRepository repository) : base(repository)
        {
            this.SupportedTemplateIds.Add(Templates.ReadJsonFilePiplineStep.ID);
        }
        protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep)
        {
            AddEndpointSettings(source, pipelineStep);
        }
        private void AddEndpointSettings(ItemModel source, PipelineStep pipelineStep)
        {
            var settings = new EndpointSettings();
            var endpointFrom = base.ConvertReferenceToModel(source, Templates.ReadJsonFilePiplineStep.Fields.EndpointFrom);
            if (endpointFrom != null)
            {
                settings.EndpointFrom = endpointFrom;
            }
            pipelineStep.Plugins.Add(settings);
        }
    }

VS-Pipline Step Processor:

[RequiredEndpointPlugins(typeof(JsonFileSettings))]
    public class ReadJsonFileStepProcessor : BaseReadDataStepProcessor
    {
        public override void Process(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            base.Process(pipelineStep, pipelineContext);
        }

        public override bool CanProcess(PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            return base.CanProcess(pipelineStep, pipelineContext);
        }

        protected override void ReadData(Endpoint endpoint, PipelineStep pipelineStep, PipelineContext pipelineContext)
        {
            if (endpoint == null)
            {
                throw new ArgumentNullException(nameof(endpoint));
            }
            if (pipelineStep == null)
            {
                throw new ArgumentNullException(nameof(pipelineStep));
            }
            if (pipelineContext == null)
            {
                throw new ArgumentNullException(nameof(pipelineContext));
            }
            var logger = pipelineContext.PipelineBatchContext.Logger;

            var settings = endpoint.GetJsonFileSettings();
            if (settings == null)
            {
                logger.Error("No text file settings are specified on the endpoint. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
                return;
            }
            if (string.IsNullOrWhiteSpace(settings.Path))
            {
                logger.Error("No path is specified on the endpoint. (pipeline step: {0}, endpoint: {1})", pipelineStep.Name, endpoint.Name);
                return;
            }

            var path = settings.Path;
            if (!Path.IsPathRooted(path))
            {
                path = string.Format("{0}{1}", AppDomain.CurrentDomain.BaseDirectory, path);
            }

            if (!File.Exists(path))
            {
                logger.Error("The path specified on the endpoint does not exist. (pipeline step: {0}, endpoint: {1}, path: {2})", pipelineStep.Name, endpoint.Name, path);
                return;
            }

            Type objectType = Type.GetType(settings.CastTypeFullName, false, true);

            Type genericListType = typeof(List<>).MakeGenericType(objectType);
            var items = (IList)Activator.CreateInstance(genericListType);

            using (StreamReader r = new StreamReader(settings.Path))
            {
                string json = r.ReadToEnd();

                JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings
                {
                    Converters = new List { new BadDateFixingConverter("yyyy-MM-dd") },
                    DateParseHandling = DateParseHandling.None
                };

                items = (IList)JsonConvert.DeserializeObject(json, genericListType, jsonSerializerSettings);

                foreach (var item in items)
                {
                    string itemNameSourcePropertyValue = item.GetPropertyValue(settings.ItemNameSourceProperty).ToString();

                    item.SetPropertyValue("ItemName", Helper.NormalizeItemName(itemNameSourcePropertyValue));
                }
            }

            var dataSettings = new IterableDataSettings(items);
            logger.Info("{0} objects were read from the file. (pipeline step: {1}, endpoint: {2})", items.Count, pipelineStep.Name, endpoint.Name);

            pipelineContext.Plugins.Add(dataSettings);
        }
    }

So this is a normal provider implementation, that use property value to read and write the values, but in this way, only the string value (regular) values will be imported correctly.
We still need to handle the other Sitecore field types.

for example, to import a Sitecore image from URL string:

1. Read the URL and download the image.
2. Upload the image to the Sitecore media library.
3. Assign the uploaded image to the target Sitecore item.

I will use the value transformer to do this as below:

SC-Image Value reader:

let the image value mapping use this reader in "Apply Mapping Pipeline Step", as below :

VS-Image Value Converter:

public class ImageValueAccessorConverter : BaseItemModelConverter
    {
        public ImageValueAccessorConverter(IItemModelRepository repository)
        : base(repository)
        {
            this.SupportedTemplateIds.Add(Templates.ImageValueReader.ID);
        }

        public override IValueReader Convert(ItemModel source)
        {
            if (source == null)
            {
                return null;
            }
            if (!this.CanConvert(source))
            {
                return null;
            }

            var uploadPath = base.GetStringValue(source, Templates.ImageValueReader.Fields.UploadPath);
            if (string.IsNullOrWhiteSpace(uploadPath))
            {
                return null;
            }
            return new ImageValueReader(uploadPath);
        }
    }

VS-Image Value Reader:

 public class ImageValueReader : IValueReader
    {
        string mediaUploadPath = string.Empty;

        public ImageValueReader(string _mediaUploadPath)
        {
            mediaUploadPath = _mediaUploadPath;
        }
        public CanReadResult CanRead(object source, DataAccessContext context)
        {
            bool canReadValue = true;
            if (context == null)
            {
                throw new ArgumentNullException("context");
            }

            canReadValue = !string.IsNullOrWhiteSpace(mediaUploadPath);

            if (canReadValue)
            {
                string imageUrl = (string)source;

                string fileName = imageUrl.Split(new string[] { "/" }, StringSplitOptions.None).LastOrDefault();

                canReadValue = !string.IsNullOrWhiteSpace(fileName);
            }

            return new CanReadResult()
            {
                CanReadValue = canReadValue
            };
        }

        public ReadResult Read(object source, DataAccessContext context)
        {
            string imageUrl = (string)source;
            ImageItemRepository imageItemRepository = new ImageItemRepository();

            string fileName = imageUrl.Split(new string[] { "/" }, StringSplitOptions.None).LastOrDefault();

            ID itemId = imageItemRepository.Create(fileName, mediaUploadPath, imageUrl);

            return new ReadResult(DateTime.UtcNow) { ReadValue = imageItemRepository.GetImageRawValue(itemId), WasValueRead = true };
        }
    }

VS-Image Item Repository:

  public class ImageItemRepository
    {
        public ID Create(string fileName, string uploadPath, string srcUrl)
        {
            try
            {
                if (string.IsNullOrWhiteSpace(uploadPath))
                {
                    return ID.Null;
                }

                string sitecoreItemName = Helper.NormalizeItemName(fileName.Split('.')[0]);
                Item mediaItem;
                var webRequest = WebRequest.Create(srcUrl.FixUrl());
                using (var webResponse = webRequest.GetResponse())
                {
                    using (var stream = webResponse.GetResponseStream())
                    {
                        if (stream != null)
                        {
                            using (var memoryStream = new MemoryStream())
                            {
                                stream.CopyTo(memoryStream);

                                var mediaCreator = new MediaCreator();
                                var options = new MediaCreatorOptions();
                                options.FileBased = false;
                                options.IncludeExtensionInItemName = false;
                                options.OverwriteExisting = true;
                                options.Versioned = false;
                                options.Destination = uploadPath + "/" + sitecoreItemName;
                                options.Database = Configuration.Factory.GetDatabase("master");

                                using (new SecurityDisabler())
                                {
                                    mediaItem = mediaCreator.CreateFromStream(memoryStream, fileName, options);

                                    return mediaItem.ID;
                                }
                            }
                        }
                    }
                }

                return ID.Null;
            }
            catch (Exception)
            {
                return ID.Null;
            }
        }

      public string GetImageRawValue(ID itemId)
        {
            return "<image mediaid=\"" + itemId.ToString() + "\" />";
        }
    }