Bulk Upload/Migrate File shares to SharePoint On-Premises and Online

There are many examples available that shows how to bulk migrate or upload files to SharePoint but none of those are providing a method to migrate with metadata.  The script below can be used to migrate complicated folder hierarchy to be uploaded to SharePoint Online or On-Premises.  The script is based on the the effort on Technet example https://gallery.technet.microsoft.com/PowerShell-Bulk-Upload-b9e9d600.  The same script can be modified to upload large amount of metadata.  You just need to export the files list to a CSV and then read values after the upload.  I will upload the other example later.

[CmdletBinding()]
param(
[Parameter(Mandatory=$false,Position=1)]
[String]$Credentials,
[Parameter(Mandatory=$True, Position=3)]
[String]$SiteURL,
[Parameter(Mandatory=$True, Position=4)]
[String]$DocLibName,
[Parameter(Mandatory=$True, Position=5)]
[String]$Folder,
[Parameter(Mandatory=$False, Position=6)]
[Switch]$Checkin,
[Parameter(Mandatory=$False, Position=7)]
[Switch]$O365
)

<#
    Add references to SharePoint client assemblies and authenticate to Office 365 site – required for CSOM

Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.dll”
Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.Runtime.dll”
#>

Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.dll” -ErrorAction SilentlyContinue
Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.Runtime.dll” -ErrorAction SilentlyContinue

Add-Type -Path ([System.Reflection.Assembly]::LoadWithPartialName(“Microsoft.SharePoint.Client”).location)
Add-Type -Path ([System.Reflection.Assembly]::LoadWithPartialName(“Microsoft.SharePoint.Client.runtime”).location)

function GetUserLookupString{
    [CmdletBinding()]
    param($context, $userString)
   
    try{
        $user = $context.Web.EnsureUser($userString)
        $context.Load($user)
        $context.ExecuteQuery()
       
        # The “proper” way would seem to be to set the user field to the user value object
        # but that does not work, so we use the formatted user lookup string instead
        #$userValue = New-Object Microsoft.SharePoint.Client.FieldUserValue
        #$userValue.LookupId = $user.Id
        $userLookupString = “{0};#{1}” -f $user.Id, $user.LoginName
    }
    catch{
        Write-Host “Unable to ensure user ‘$($userString)’.” -ErrorAction SilentlyContinue
        $userLookupString = $null
    }
   
    return $userLookupString
}

<#
    Define Functions
#>

<#
    Upload File – This function performs the actual file upload
#>
function UploadFile($DestinationFolder, $File)
{
    #Get the datastream of the file, assign it to a variable
    $FileStream = New-Object IO.FileStream($File.FullName,[System.IO.FileMode]::Open)

    #Create an instance of a FileCreationInformation object
    $FileCreationInfo = New-Object Microsoft.SharePoint.Client.FileCreationInformation

    #Indicate whether or not you would like to overwrite files in the event of a conflict
    $FileCreationInfo.Overwrite = $True

    #Make the datastream of the file you wish to create equal to the datastream of the source file
    $FileCreationInfo.ContentStream = $FileStream

    #Make the URL of the file equal to the $File variable which was passed to the function.  This will be equal to the source file name
    $FileCreationInfo.url = $File

    #Add the file to the destination folder which was passed to the function, using the FileCreationInformation supplied.  Assign this to a variable so that it can be loaded into context.
    $Upload = $DestinationFolder.Files.Add($FileCreationInfo)
    if($Checkin)
    {
        $Context.Load($Upload)
        $Context.ExecuteQuery()
        if($Upload.CheckOutType -ne “none”)
        {
            $Upload.CheckIn(“Checked in by Administrator”, [Microsoft.SharePoint.Client.CheckinType]::MajorCheckIn)
        }
    }
    $Context.Load($Upload)
    $Context.ExecuteQuery()

    Write-Host “Adding Metadata to File Item…$($File.FullName)”
            $listItem = $Upload.ListItemAllFields
            $Context.Load($listItem)
            #$Context.ExecuteQuery()
            $Context.ExecuteQuery()
   
            $Item = $List.GetItemById($listItem.Id);
            $Context.Load($Item)

            $Owner = $File.GetAccessControl().Owner
            $LastModifiedBy =Get-ChildItem $File.FullName | Sort {$_.LastWriteTime} | select -last 1 | foreach {$a=$_;$b=Get-Acl $_.FullName; Add-Member -InputObject $b -Name “LastWriteTime” -MemberType NoteProperty -Value $a.LastWriteTime;$b}
           
            if($Owner -eq “BUILTIN\Administrators”)
            {
                $Author = “hp\spadmin”
            }

            if($LastModifiedBy.Owner -eq “BUILTIN\Administrators”)
            {
                $Editor = “hp\spadmin”
            }
     
            $Author = GetUserLookupString $Context $Author
            if($Author)
            {
                $Item[“Author”] = $Author
            }
            $Editor = GetUserLookupString $Context $Editor
            if($Editor)
            {
                $Item[“Editor”] = $Editor
            }
       
            [System.DateTime]$CreatedOnDate = $File.CreationTime
            [System.DateTime]$ModifiedOnDate = $File.LastWriteTime

            $Item[“Created”] = $CreatedOnDate
            $Item[“Modified”] = $ModifiedOnDate
            try
            {
                $Item.Update()
                $Context.ExecuteQuery()
            }
            catch
            { Write-Host “Exception Occured in reading list item properties… But script will continue” }
           
}

<#
    Create Folder Function.
#>
function PopulateFolder($ListRootFolder, $FolderRelativePath, $FolderFullPath)
{
    #split the FolderRelativePath passed into chunks (between the backslashes) so that we can check if the folder structure exists
    $PathChunks = $FolderRelativePath.substring(1).split(“\”)

    #Make sure we start with a fresh WorkingFolder for every folder passed to the function
    if($WorkingFolder)
    {
        Remove-Variable WorkingFolder
    }

    #Start with the root folder of the list, load this into context
    $WorkingFolder = $ListRootFolder
    $Context.load($WorkingFolder)
    $Context.ExecuteQuery()

    #Load the folders of the current working folder into context
    $Context.load(($WorkingFolder.folders))
    $Context.executeQuery()

    #Set the FileSource folder equal to the absolute path of the folder that passed to the function
    $FileSource = $Folder + $FolderRelativePath
   
    #Loop through the folder chunks, ensuring that the correct folder hierarchy exists in the destination
    foreach($Chunk in $PathChunks)
    {
        #Check to find out if a subfolder exists in the current folder that matches the patch chunk being evaluated
        if($WorkingFolder.folders | ? {$_.name -eq $Chunk})
        {
            #Log the status to the PowerShell host window
            Write-Host “Folder $Chunk Exists in” $WorkingFolder.name -ForegroundColor Green

            #Since we will be evaluating other chunks in the path, set the working folder to the current folder and load this into context.
            $WorkingFolder = $WorkingFolder.folders | ? {$_.name -eq $Chunk}
            $Context.load($WorkingFolder)
            $Context.load($WorkingFolder.folders)
            $Context.ExecuteQuery()

        }
        else
        {
            #If the folder doesn’t exist, Log a message indicating that the folder doesn’t exist, and another message indicating that it is being created
            Write-Host “Folder $Chunk Does Not Exist in” $WorkingFolder.name -ForegroundColor Yellow
            Write-Host “Creating Folder $Chunk in” $WorkingFolder.name -ForegroundColor Green
           
            #Load the working folder into context and create a subfolder with a name equal to the chunk being evaluated, and load this into context
            $Context.load($WorkingFolder)
            $Context.load($WorkingFolder.folders)
            $Context.ExecuteQuery()

            $WorkingFolder= $WorkingFolder.folders.add($Chunk)
            $Context.load($WorkingFolder)
            $Context.load($WorkingFolder.folders)
            $Context.ExecuteQuery()

            #$FolderRelativePath = “C:\Demo\Corporate Docs”
            $LocalFolder = Get-Item -Path $FolderFullPath

            $Web = $Context.Web
            $Context.Load($Web)
            $Context.ExecuteQuery()

            $SPFolder = $Web.GetFolderByServerRelativeUrl($WorkingFolder.ServerRelativeUrl)
            $Context.Load($SPFolder)
            $Context.Load($SPFolder.ListItemAllFields)
            $Context.ExecuteQuery()
            $FolderItemId = $SPFolder.ListItemAllFields.Id
            $FItem = $List.GetItemById($FolderItemId)
            $Context.Load($FItem)
            $Context.ExecuteQuery()

            #$Query = New-Object Microsoft.SharePOint.Client.CamlQuery
            #$Query.ViewXml = “<View Scope=’RecursiveAll’><Query><Where><And><Eq><FieldRef Name=’ContentType’/><Value Type=’Text’>Folder</Value></Eq><Eq><FieldRef Name=’FileLeafRef’/><Value Type=’Text’>” + $Chunk + “</Value></Eq></And></Where></Query></View>”

            $FItem[“Created”] = $LocalFolder.CreationTime.ToString()
            $FItem[“Modified”] = $LocalFolder.LastWriteTime.ToString()
            $CreatedBy = $LocalFolder.GetAccessControl().Owner
            if($CreatedBy -eq “BUILTIN\Administrators”)
            {
                $CreatedBy = “hp\spadmin”
            }

            $Author = GetUserLookupString $Context $CreatedBy
            if($Author)
            {
                $FItem[“Editor”] = $Author
                $FItem[“Author”] = $Author
            }
            $Context.ExecuteQuery()

            try
            {
                $FItem.Update()
                $Context.ExecuteQuery()
            }
            catch
            { Write-Host “Exception Occured in updating folder properties… But script will continue” }

        }

    }

    #Folder is confirmed existing or created – now it’s time to list all files in the source folder, and assign this to a variable
    $FilesInFolder = Get-ChildItem -Path $FileSource | ? {$_.psIsContainer -eq $False}
   
    #For each file in the source folder being evaluated, call the UploadFile function to upload the file to the appropriate location
    Foreach ($File in ($FilesInFolder))
    {

        #Notify the operator that the file is being uploaed to a specific location
        Write-Host “Uploading file ” $file.Name “to” $WorkingFolder.name -ForegroundColor Cyan

        #Upload the file
        UploadFile $WorkingFolder $File

    }
   
   
   
}

<#
    Bind your context to the site collection
#>
$Context = New-Object Microsoft.SharePoint.Client.ClientContext($SiteURL)

<#
    Create a credential object using the username and password supplied
#>

if($O365)
{
    $Creds = New-Object Microsoft.SharePoint.Client.SharePointOnlineCredentials($Credentials.UserName,$Credentials.Password)
}
else
{
    $Creds = [System.Net.CredentialCache]::DefaultCredentials
    #$Creds = New-Object System.Net.NetworkCredential($Credentials.UserName,$Credentials.Password)
}
#if($O365)
#{
#    $Creds = New-Object Microsoft.SharePoint.Client.SharePointOnlineCredentials($UserName,(ConvertTo-SecureString $Password -AsPlainText -Force))
#}
#else
#{
#    #$Creds = New-Object System.Net.NetworkCredential($UserName, (ConvertTo-SecureString $Password -AsPlainText -Force))
#}

<#
    Set the credentials that are used in the context.
#>
$Context.Credentials = $Creds

<#
    Retrieve the library, and load it into the context
#>
$List = $Context.Web.Lists.GetByTitle($DocLibName)
$Context.Load($List)
$Context.ExecuteQuery()

$List.Fields.GetByInternalNameOrTitle(“Author”).ReadOnlyField = $true
$List.Fields.GetByInternalNameOrTitle(“Editor”).ReadOnlyField = $true
$List.Fields.GetByInternalNameOrTitle(“Created”).ReadOnlyField = $true
$List.Fields.GetByInternalNameOrTitle(“Modified”).ReadOnlyField = $true
#$List.EnableVersioning = $True
$List.Update()
$Context.ExecuteQuery()

#Get a recursive list of all folders beneath the folder supplied by the operator
$AllFolders = Get-ChildItem -Recurse -Path $Folder |? {$_.psIsContainer -eq $True}

#Get a list of all files that exist directly at the root of the folder supplied by the operator
$FilesInRoot = Get-ChildItem -Path $Folder | ? {$_.psIsContainer -eq $False}

#Upload all files in the root of the folder supplied by the operator
Foreach ($File in ($FilesInRoot))
{

    #Notify the operator that the file is being uploaded to a specific location
    Write-Host “Uploading file ” $File.Name “to” $DocLibName -ForegroundColor Cyan

    #Upload the file
    UploadFile($list.RootFolder) $File
   

}

#Loop through all folders (recursive) that exist within the folder supplied by the operator
foreach($CurrentFolder in $AllFolders)
{
    #Set the FolderRelativePath by removing the path of the folder supplied by the operator from the fullname of the folder
    $FolderRelativePath = ($CurrentFolder.FullName).Substring($Folder.Length)
   
    #Call the PopulateFolder function for the current folder, which will ensure that the folder exists and upload all files in the folder to the appropriate location
    PopulateFolder ($list.RootFolder) $FolderRelativePath $CurrentFolder.FullName
}

#Calling the script for SharePoint on-premises

$Creds = Get-Credential
.\BulkUploadSharePointCSOM.ps1 -SiteURL “https://portal.hp.com” -DocLibName “Metadata” -Folder “C:\Demo”

#Calling the script for SharePoint online

$Creds = Get-Credential
.\BulkUploadSharePointCSOM.ps1 -SiteURL “https://portal.hp.com” -DocLibName “Metadata” -Folder “C:\Demo” -Credentials $Creds -Checkin -O365

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s