ASP.NET Sample code to add, delete, modify and query Hadoop

Time:2021-4-12

This paper introduces ASP.NET The sample code for adding, deleting, modifying and querying Hadoop is shared with you as follows:

packages.config


<?xml version="1.0" encoding="utf-8"?>
<packages>
 <package version="4.0.20505.0" targetFramework="net46" />
 <package version="5.2.0" targetFramework="net46" />
 <package version="5.2.0" targetFramework="net46" />
 <package version="0.12.5126.42915" targetFramework="net46" />
 <package version="2.0.20505.0" targetFramework="net46" />
 <package version="1.8.0.0" targetFramework="net46" />
 <package version="4.5.11" targetFramework="net46" />
 <package version="5.2.0" targetFramework="net46" />
 <package version="2.0.4.1" targetFramework="net46" />
</packages>

HDFSAccess.cs

using Microsoft.Hadoop.WebHDFS;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;

namespace Physical
{
  public sealed class HDFSAccess
  {
    private readonly WebHDFSClient webHDFSClient;

    public HDFSAccess(string uriString, string userName)
    {
      this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName);
    }

    public List<string> GetDirectories(string path)
    {
      var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

      return directoryStatus.Directories.Select(d => d.PathSuffix).ToList();
    }

    public List<string> GetFiles(string path)
    {
      var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

      return directoryStatus.Files.Select(d => d.PathSuffix).ToList();
    }

    public bool CreateDirectory(string path)
    {
      //When the incoming path does not contain the root directory, it will be under the root directory by default
      return this.webHDFSClient.CreateDirectory(path).Result;
    }

    public bool DeleteDirectory(string path)
    {
      //When the incoming path does not contain the root directory, it will be under the root directory by default
      return this.webHDFSClient.DeleteDirectory(path).Result;
    }

    public string CreateFile(string localFile, string remotePath)
    {
      //When the incoming remote path does not contain the root directory, it will be under the root directory by default
      return this.webHDFSClient.CreateFile(localFile, remotePath).Result;
    }

    public bool DeleteFile(string path)
    {
      //When the incoming path does not contain the root directory, it will be under the root directory by default
      return this.webHDFSClient.DeleteDirectory(path).Result;
    }

    public HttpResponseMessage OpenFile(string path)
    {
      //When the incoming path does not contain the root directory, it will be under the root directory by default
      return this.webHDFSClient.OpenFile(path).Result;
    }
  }
}

Program.cs

using Physical;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace MyTest
{
  class Program
  {
    //The entry endpoint of HDFS cluster client is set on the host
    //Default endpoint: http: // [host name]: 50070
    //Default account number: Wu
    private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu");

    static void Main(string[] args)
    {
      GetDirectoriesTest();
      Console.WriteLine("----------------------------------------------");
      GetFilesTest();
      Console.WriteLine("----------------------------------------------");
      DirectoryTest();
      Console.WriteLine("----------------------------------------------");
      FileTest();
      Console.WriteLine("----------------------------------------------");
      OpenFileTest();
      Console.ReadKey();
    } 

    public void TestCleanup()
    {
      //Get root folder
      var directories = access.GetDirectories(@"/");

      //Remove the default directory: TMP, user directory
      foreach (var directory in directories)
      {
        if ("tmp".Equals(directory) || "user".Equals(directory))
        {
          continue;
        }
        else
        {
          access.DeleteDirectory(directory);
        }
      }

      //Get root file
      var files = access.GetFiles(@"/");

      //Remove all files
      foreach (var file in files)
      {
        access.DeleteFile(file);
      }

      //Remove openFile transfer file
      File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"));
    }

    //_ Incoming root_ Expected return default directory
    public static void GetDirectoriesTest()
    {
      //There are two directories under the default root directory: TMP and user
      var expected = new List<string>() { "tmp", "user", };

      var actual = access.GetDirectories(@"/");

      foreach (var item in actual)
      {
        Console.WriteLine(item);
      }

    }

    //_ Incoming root_ Expected return empty set
    public static void GetFilesTest()
    {
      //There are no files in the default root directory
      var expected = new List<string>();

      var actual = access.GetFiles(@"/");
      foreach (var item in actual)
      {
        Console.WriteLine(item);
      }

    }

    //_ Set up zzz directory_ Expected success_ The zzz directory is expected under the root directory_ Delete zzz directory_ Expected success_ There is no zzz directory in the expected root directory
    public static void DirectoryTest()
    {
      var directoryName = "zzz";

      //Set up zzz directory
      var boolCreateDirectory = access.CreateDirectory(directoryName);

      Console.WriteLine (create zzz directory)_ Expected success "+ boolcreatedirectory)";

      //Set up zzz directory_预期成功_预期根目录下有zzz目录
      //There are three directories under the default root directory: TMP, user and zzz
      var expectedCreateDirectory = new List<string>() { "tmp", "user", directoryName, };

      var actualCreateDirectory = access.GetDirectories(@"/");


      foreach (var item in actualCreateDirectory)
      {
        Console.WriteLine(item);
      }

      Console.WriteLine("********************************************");
      //Delete zzz directory
      var boolDeleteDirectory = access.DeleteDirectory(directoryName);

      Console.WriteLine (delete zzz directory)_ Expected success "+ booldeletedirectory)";


      //Delete zzz directory_预期成功_预期根目录下无zzz目录
      //There are two directories under the default root directory: TMP and user
      var expectedDeleteDirectory = new List<string>() { "tmp", "user", };

      var actualDeleteDirectory = access.GetDirectories(@"/");

      foreach (var item in actualDeleteDirectory)
      {
        Console.WriteLine(item);
      }
    }

    //_ Establish test file_ Expected test file in root directory_ Delete test file_ Expected success_ There is no test file in the expected root directory
    public static void FileTest()
    {
      var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");

      var remotePath = "Test.jpg";

      //Establish test file
      var boolCreateFile = access.CreateFile(localFile, remotePath);

      //Establish test file_预期根目录下有Test档案
      var expectedCreateFile = new List<string>() { remotePath, };

      var actualCreateFile = access.GetFiles(@"/");

      foreach (var item in actualCreateFile)
      {
        Console.WriteLine(item);
      }
      Console.WriteLine("********************************************");
      //Delete test file
      var boolDeleteFile = access.DeleteDirectory(remotePath);

      Console.WriteLine (delete test file)_ Expected success "+ booldeletefile)";

      //Delete test file_预期成功_预期根目录下无Test档案
      var expectedDeleteFile = new List<string>();

      var actualDeleteFile = access.GetFiles(@"/");

      foreach (var item in actualDeleteFile)
      {
        Console.WriteLine(item);
      }
    }

    //_ Establish test file_ Expected test file in root directory_ Get test file_ Expected success_ It is expected that the test file stream will be returned and transferred successfully
    public static void OpenFileTest()
    {
      var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");
      var remotePath = "Test.jpg";
      var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg");

      Console.WriteLine("saveFile:" + saveFile);
      Console.WriteLine("********************************************");
      //Establish test file
      var boolCreateFile = access.CreateFile(localFile, remotePath);

      //Establish test file_预期根目录下有Test档案
      var expectedCreateFile = new List<string>() { remotePath, };

      var actualCreateFile = access.GetFiles(@"/");

      foreach (var item in actualCreateFile)
      {
        Console.WriteLine(item);
      }


      Console.WriteLine("********************************************");
      //Get test file
      var response = access.OpenFile(remotePath);

      //Get test file_预期成功
      response.EnsureSuccessStatusCode();

      //Get test file_预期成功_预期回传Test档案Stream并转存成功
      using (var fs = File.Create(saveFile))
      {
        response.Content.CopyToAsync(fs).Wait();
      }

      Console.WriteLine(File.Exists(saveFile));

    }
  }
}

The running results are as follows:

The above is the whole content of this article, I hope to help you learn, and I hope you can support developer more.